diff --git a/docs/my-website/docs/providers/vllm.md b/docs/my-website/docs/providers/vllm.md index 5472f0602f..1a37f2f10e 100644 --- a/docs/my-website/docs/providers/vllm.md +++ b/docs/my-website/docs/providers/vllm.md @@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM. | Property | Details | |-------|-------| | Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) | -| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) | +| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) | | Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) | -| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank` | +| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` | # Quick Start diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8dc3061460..ce054b91cc 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1,6 +1,5 @@ import asyncio import json -import re import time import traceback import uuid @@ -9,6 +8,9 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union import litellm from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, +) from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, @@ -274,49 +276,6 @@ def _handle_invalid_parallel_tool_calls( return tool_calls -def _parse_content_for_reasoning( - message_text: Optional[str], -) -> Tuple[Optional[str], Optional[str]]: - """ - Parse the content for reasoning - - Returns: - - reasoning_content: The content of the reasoning - - content: The content of the message - """ - if not message_text: - return None, message_text - - reasoning_match = re.match( - r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL - ) - - if reasoning_match: - return reasoning_match.group(1), reasoning_match.group(2) - - return None, message_text - - -def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: - """ - Extract reasoning content and main content from a message. - - Args: - message (dict): The message dictionary that may contain reasoning_content - - Returns: - tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) - """ - message_content = message.get("content") - if "reasoning_content" in message: - return message["reasoning_content"], message["content"] - elif "reasoning" in message: - return message["reasoning"], message["content"] - elif isinstance(message_content, str): - return _parse_content_for_reasoning(message_content) - return None, message_content - - class LiteLLMResponseObjectHandler: @staticmethod def convert_to_image_response( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a99883ef7b..19d5932ff2 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -14,6 +14,7 @@ from typing import ( Literal, Mapping, Optional, + Tuple, Union, cast, ) @@ -869,3 +870,63 @@ def convert_prefix_message_to_non_prefix_messages( else: new_messages.append(message) return new_messages + + +def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: + """ + Extract reasoning content and main content from a message. + + Args: + message (dict): The message dictionary that may contain reasoning_content + + Returns: + tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) + """ + message_content = message.get("content") + if "reasoning_content" in message: + return message["reasoning_content"], message["content"] + elif "reasoning" in message: + return message["reasoning"], message["content"] + elif isinstance(message_content, str): + return _parse_content_for_reasoning(message_content) + return None, message_content + + +def _parse_content_for_reasoning( + message_text: Optional[str], +) -> Tuple[Optional[str], Optional[str]]: + """ + Parse the content for reasoning + + Returns: + - reasoning_content: The content of the reasoning + - content: The content of the message + """ + if not message_text: + return None, message_text + + reasoning_match = re.match( + r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL + ) + + if reasoning_match: + return reasoning_match.group(1), reasoning_match.group(2) + + return None, message_text + + +def extract_images_from_message(message: AllMessageValues) -> List[str]: + """ + Extract images from a message + """ + images = [] + message_content = message.get("content") + if isinstance(message_content, list): + for m in message_content: + image_url = m.get("image_url") + if image_url: + if isinstance(image_url, str): + images.append(image_url) + elif isinstance(image_url, dict) and "url" in image_url: + images.append(image_url["url"]) + return images diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 179b8d0fb0..3574996e48 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx @@ -23,12 +23,13 @@ else: class AudioTranscriptionRequestData: """ Structured data for audio transcription requests. - + Attributes: data: The request data (form data for multipart, json data for regular requests) files: Optional files dict for multipart form data content_type: Optional content type override """ + data: Union[dict, bytes] files: Optional[dict] = None content_type: Optional[str] = None @@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> Union[AudioTranscriptionRequestData, Dict]: + ) -> AudioTranscriptionRequestData: raise NotImplementedError( "AudioTranscriptionConfig needs a request transformation for audio transcription models" ) - - def transform_audio_transcription_response( self, raw_response: httpx.Response, @@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): raise NotImplementedError( "AudioTranscriptionConfig does not need a response transformation for audio transcription models" ) - def get_provider_specific_params( self, @@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): provider_specific_params[key] = value return provider_specific_params - + def _should_exclude_param( self, param_name: str, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fda9220ff7..e3d65be8bb 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -14,7 +14,7 @@ from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -397,7 +397,11 @@ class AmazonConverseConfig(BaseConfig): for param, value in non_default_params.items(): if param == "response_format" and isinstance(value, dict): optional_params = self._translate_response_format_param( - value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled + value=value, + model=model, + optional_params=optional_params, + non_default_params=non_default_params, + is_thinking_enabled=is_thinking_enabled, ) if param == "max_tokens" or param == "max_completion_tokens": optional_params["maxTokens"] = value @@ -446,11 +450,11 @@ class AmazonConverseConfig(BaseConfig): ) return optional_params - + def _translate_response_format_param( - self, - value: dict, - model: str, + self, + value: dict, + model: str, optional_params: dict, non_default_params: dict, is_thinking_enabled: bool, @@ -504,7 +508,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["json_mode"] = True if non_default_params.get("stream", False) is True: optional_params["fake_stream"] = True - + return optional_params def update_optional_params_with_thinking_tokens( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index d7ceec1f1c..0fe84b0ce0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast from httpx import Response from litellm import verbose_logger -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a4f40e90df..d691549bc6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -118,7 +118,6 @@ class BaseLLMHTTPHandler: response: Optional[httpx.Response] = None for i in range(max(max_retry_on_unprocessable_entity_error, 1)): try: - response = await async_httpx_client.post( url=api_base, headers=headers, @@ -2221,7 +2220,9 @@ class BaseLLMHTTPHandler: if isinstance(transformed_request, dict) and "method" in transformed_request: # Handle pre-signed requests (e.g., from Bedrock S3 uploads) - upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())( + upload_response = getattr( + sync_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2233,8 +2234,8 @@ class BaseLLMHTTPHandler: # Handle traditional file uploads # Ensure transformed_request is a string for httpx compatibility if isinstance(transformed_request, bytes): - transformed_request = transformed_request.decode('utf-8') - + transformed_request = transformed_request.decode("utf-8") + # Use the HTTP method specified by the provider config http_method = provider_config.file_upload_http_method.upper() if http_method == "PUT": @@ -2314,7 +2315,7 @@ class BaseLLMHTTPHandler: ) else: async_httpx_client = client - + ######################################################### # Debug Logging ######################################################### @@ -2330,7 +2331,9 @@ class BaseLLMHTTPHandler: if isinstance(transformed_request, dict) and "method" in transformed_request: # Handle pre-signed requests (e.g., from Bedrock S3 uploads) - upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())( + upload_response = await getattr( + async_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2342,8 +2345,8 @@ class BaseLLMHTTPHandler: # Handle traditional file uploads # Ensure transformed_request is a string for httpx compatibility if isinstance(transformed_request, bytes): - transformed_request = transformed_request.decode('utf-8') - + transformed_request = transformed_request.decode("utf-8") + # Use the HTTP method specified by the provider config http_method = provider_config.file_upload_http_method.upper() if http_method == "PUT": @@ -2468,9 +2471,14 @@ class BaseLLMHTTPHandler: sync_httpx_client = client try: - if isinstance(transformed_request, dict) and "method" in transformed_request: + if ( + isinstance(transformed_request, dict) + and "method" in transformed_request + ): # Handle pre-signed requests (e.g., from Bedrock with AWS auth) - batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())( + batch_response = getattr( + sync_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2500,8 +2508,11 @@ class BaseLLMHTTPHandler: ) # Store original request for response transformation - litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data} - + litellm_params_with_request = { + **litellm_params, + "original_batch_request": create_batch_data, + } + return provider_config.transform_create_batch_response( model=model, raw_response=batch_response, @@ -2531,7 +2542,7 @@ class BaseLLMHTTPHandler: ) else: async_httpx_client = client - + ######################################################### # Debug Logging ######################################################### @@ -2546,9 +2557,14 @@ class BaseLLMHTTPHandler: ) try: - if isinstance(transformed_request, dict) and "method" in transformed_request: + if ( + isinstance(transformed_request, dict) + and "method" in transformed_request + ): # Handle pre-signed requests (e.g., from Bedrock with AWS auth) - batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())( + batch_response = await getattr( + async_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2578,8 +2594,11 @@ class BaseLLMHTTPHandler: ) # Store original request for response transformation (for async version) - litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}} - + litellm_params_with_request = { + **litellm_params, + "original_batch_request": create_batch_data or {}, + } + return provider_config.transform_create_batch_response( model=model, raw_response=batch_response, diff --git a/litellm/llms/hosted_vllm/transcriptions/transformation.py b/litellm/llms/hosted_vllm/transcriptions/transformation.py new file mode 100644 index 0000000000..5eeb892d84 --- /dev/null +++ b/litellm/llms/hosted_vllm/transcriptions/transformation.py @@ -0,0 +1,72 @@ +""" +Transformation logic for Hosted VLLM rerank +""" + +from typing import Optional, Union + +import httpx + +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai.transcriptions.whisper_transformation import ( + OpenAIWhisperAudioTranscriptionConfig, +) +from litellm.types.utils import FileTypes + + +class HostedVLLMAudioTranscriptionError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: Optional[Union[dict, httpx.Headers]] = None, + ): + super().__init__(status_code=status_code, message=message, headers=headers) + + +class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): + def __init__(self) -> None: + pass + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base: + # Remove trailing slashes and ensure clean base URL + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/audio/transcriptions"): + api_base = f"{api_base}/v1/audio/transcriptions" + return api_base + raise ValueError("api_base must be provided for Hosted VLLM rerank") + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + """ + Transform the audio transcription request + """ + + data = {"model": model, "file": audio_file, **optional_params} + + if "response_format" not in data or ( + data["response_format"] == "text" or data["response_format"] == "json" + ): + data["response_format"] = ( + "verbose_json" # ensures 'duration' is received - used for cost calculation + ) + + return AudioTranscriptionRequestData( + data=data, + ) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index ee0d3acef7..3527a57921 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,9 +16,18 @@ from httpx._models import Headers, Response from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, + convert_content_list_to_str, + extract_images_from_message, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction +from litellm.types.llms.ollama import ( + OllamaChatCompletionMessage, + OllamaToolCall, + OllamaToolCallFunction, +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantToolCall, @@ -299,7 +308,23 @@ class OllamaChatConfig(BaseConfig): ) new_tools.append(ollama_tool_call) cast(dict, m)["tool_calls"] = new_tools - new_messages.append(m) + reasoning_content, parsed_content = _extract_reasoning_content( + cast(dict, m) + ) + content_str = convert_content_list_to_str(cast(AllMessageValues, m)) + images = extract_images_from_message(cast(AllMessageValues, m)) + + ollama_message = OllamaChatCompletionMessage( + role=cast(str, m.get("role")), + ) + if reasoning_content is not None: + ollama_message["thinking"] = reasoning_content + if content_str is not None: + ollama_message["content"] = content_str + if images is not None: + ollama_message["images"] = images + + new_messages.append(ollama_message) # Load Config config = self.get_config() @@ -361,7 +386,7 @@ class OllamaChatConfig(BaseConfig): del response_json_message["thinking"] elif response_json_message.get("content") is not None: # parse reasoning content from content - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 71bcf0bb3f..bfb0b7f187 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -229,7 +229,7 @@ class OllamaConfig(BaseConfig): model = model.split("/", 1)[1] api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" api_key = self.get_api_key() - headers = { "Authorization": f"Bearer {api_key}" } if api_key else {} + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} try: response = litellm.module_level_client.post( @@ -279,7 +279,7 @@ class OllamaConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index 796e10f515..34621c44e2 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -1,5 +1,8 @@ from typing import List +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams from litellm.types.utils import FileTypes @@ -27,8 +30,12 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> dict: + ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request """ - return {"model": model, "file": audio_file, **optional_params} + data = {"model": model, "file": audio_file, **optional_params} + + return AudioTranscriptionRequestData( + data=data, + ) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 4fe48dd3c6..19b303bb96 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import Optional, Union, cast import httpx from openai import AsyncOpenAI, OpenAI @@ -34,6 +34,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: + raw_response = ( await openai_aclient.audio.transcriptions.with_raw_response.create( **data, timeout=timeout @@ -93,15 +94,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): Handle audio transcription request """ if provider_config is not None: - data = provider_config.transform_audio_transcription_request( + transformed_data = provider_config.transform_audio_transcription_request( model=model, audio_file=audio_file, optional_params=optional_params, litellm_params=litellm_params, ) - if not isinstance(data, dict): - raise ValueError("OpenAI transformation route requires a dict") + data = cast(dict, transformed_data.data) else: data = {"model": model, "file": audio_file, **optional_params} diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index c0ccc71579..fa507e1bc2 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -1,8 +1,9 @@ from typing import List, Optional, Union -from httpx import Headers +from httpx import Headers, Response from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -11,12 +12,40 @@ from litellm.types.llms.openai import ( AllMessageValues, OpenAIAudioTranscriptionOptionalParams, ) -from litellm.types.utils import FileTypes +from litellm.types.utils import FileTypes, TranscriptionResponse from ..common_utils import OpenAIError class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + ## get the api base, attach the endpoint - v1/audio/transcriptions + # strip trailing slash if present + api_base = api_base.rstrip("/") if api_base else "" + + # if endswith "/v1" + if api_base and api_base.endswith("/v1"): + api_base = f"{api_base}/audio/transcriptions" + else: + api_base = f"{api_base}/v1/audio/transcriptions" + + return api_base or "" + def get_supported_openai_params( self, model: str ) -> List[OpenAIAudioTranscriptionOptionalParams]: @@ -72,21 +101,22 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> dict: + ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request """ - data = {"model": model, "file": audio_file, **optional_params} if "response_format" not in data or ( data["response_format"] == "text" or data["response_format"] == "json" ): - data[ - "response_format" - ] = "verbose_json" # ensures 'duration' is received - used for cost calculation + data["response_format"] = ( + "verbose_json" # ensures 'duration' is received - used for cost calculation + ) - return data + return AudioTranscriptionRequestData( + data=data, + ) def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, Headers] @@ -96,3 +126,25 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): message=error_message, headers=headers, ) + + def transform_audio_transcription_response( + self, + raw_response: Response, + ) -> TranscriptionResponse: + try: + raw_response_json = raw_response.json() + except Exception as e: + raise ValueError( + f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}" + ) + + if any( + key in raw_response_json + for key in TranscriptionResponse.model_fields.keys() + ): + return TranscriptionResponse(**raw_response_json) + else: + raise ValueError( + "Invalid response format. Received response does not match the expected format. Got: ", + raw_response_json, + ) diff --git a/litellm/main.py b/litellm/main.py index 6c81d3eded..c910c88d17 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5267,7 +5267,10 @@ def transcription( model_response = litellm.utils.TranscriptionResponse() model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, ) # type: ignore if dynamic_api_key is not None: @@ -5283,6 +5286,7 @@ def transcription( custom_llm_provider=custom_llm_provider, **non_default_params, ) + litellm_params_dict = get_litellm_params(**kwargs) litellm_logging_obj.update_environment_variables( @@ -5347,9 +5351,8 @@ def transcription( max_retries=max_retries, litellm_params=litellm_params_dict, ) - elif ( - custom_llm_provider == "openai" - or custom_llm_provider in litellm.openai_compatible_providers + elif custom_llm_provider == "openai" or ( + custom_llm_provider in litellm.openai_compatible_providers ): api_base = ( api_base @@ -5364,6 +5367,7 @@ def transcription( or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) # set API KEY + api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") # type: ignore response = openai_audio_transcriptions.audio_transcriptions( model=model, @@ -5380,10 +5384,7 @@ def transcription( provider_config=provider_config, litellm_params=litellm_params_dict, ) - elif custom_llm_provider in [ - LlmProviders.DEEPGRAM.value, - LlmProviders.ELEVENLABS.value, - ]: + elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( model=model, audio_file=file, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 2fac2781a1..8b9f81b41a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -9,4 +9,12 @@ model_list: model: openai/* - model_name: xai-grok-3 litellm_params: - model: xai/grok-3 \ No newline at end of file + model: xai/grok-3 + - model_name: hosted_vllm/whisper-v3 + litellm_params: + model: hosted_vllm/whisper-v3 + api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" + api_key: dummy + + + diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 5bdddfc4c6..8f2ba37a9d 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -93,7 +93,7 @@ async def _upsert_budget_and_membership( create_data["tpm_limit"] = tpm_limit if rpm_limit is not None: create_data["rpm_limit"] = rpm_limit - + new_budget = await tx.litellm_budgettable.create( data=create_data, include={"team_membership": True}, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a7f15cc6c1..8dac7685f1 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -925,6 +925,15 @@ async def prepare_key_update_data( detail="team_id is required for service account keys. Please specify `team_id` in the request body.", ) non_default_values = {} + # ADD METADATA FIELDS + # Set Management Endpoint Metadata Fields + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) for k, v in data_json.items(): if ( k in LiteLLM_ManagementEndpoint_MetadataFields @@ -1137,6 +1146,9 @@ async def update_key_fn( change_initiated_by=user_api_key_dict, llm_router=llm_router, ) + + # Set Management Endpoint Metadata Fields + non_default_values = await prepare_key_update_data( data=data, existing_key_row=existing_key_row ) diff --git a/litellm/router.py b/litellm/router.py index d6964745e7..519c7797da 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4414,7 +4414,7 @@ class Router: return tpm_key except Exception as e: - verbose_router_logger.exception( + verbose_router_logger.debug( "litellm.router.Router::deployment_callback_on_success(): Exception occured - {}".format( str(e) ) @@ -4562,8 +4562,10 @@ class Router: parent_otel_span=parent_otel_span, ttl=RoutingArgs.ttl.value, ) - - def _get_metadata_variable_name_from_kwargs(self, kwargs: dict) -> Literal["metadata", "litellm_metadata"]: + + def _get_metadata_variable_name_from_kwargs( + self, kwargs: dict + ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data @@ -5672,11 +5674,11 @@ class Router: ) if supported_openai_params is None: supported_openai_params = [] - + # Get mode from database model_info if available, otherwise default to "chat" db_model_info = model.get("model_info", {}) mode = db_model_info.get("mode", "chat") - + model_info = ModelMapInfo( key=model_group, max_tokens=None, @@ -6802,7 +6804,9 @@ class Router: model=model, request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, - metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + metadata_variable_name=self._get_metadata_variable_name_from_kwargs( + request_kwargs + ), ) if len(healthy_deployments) == 0: diff --git a/litellm/types/llms/ollama.py b/litellm/types/llms/ollama.py index 9d71904caa..d8f20de514 100644 --- a/litellm/types/llms/ollama.py +++ b/litellm/types/llms/ollama.py @@ -27,3 +27,12 @@ class OllamaToolCall(TypedDict): class OllamaVisionModelObject(TypedDict): prompt: str images: List[str] + + +class OllamaChatCompletionMessage(TypedDict, total=False): + role: Required[str] + content: str + thinking: str + images: List[str] + tool_calls: List[OllamaToolCall] + tool_name: str diff --git a/litellm/utils.py b/litellm/utils.py index 5cbbfd34a6..31b94984e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -107,7 +107,6 @@ from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_s from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( LiteLLMResponseObjectHandler, _handle_invalid_parallel_tool_calls, - _parse_content_for_reasoning, convert_to_model_response_object, convert_to_streaming_response, convert_to_streaming_response_async, @@ -122,6 +121,9 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ResponseMetadata, ) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, +) from litellm.litellm_core_utils.redact_messages import ( LiteLLMLoggingObject, redact_message_input_output_from_logging, @@ -2426,6 +2428,8 @@ def get_optional_params_transcription( # retrieve all parameters passed to the function passed_params = locals() + + passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") custom_llm_provider = passed_params.pop("custom_llm_provider") drop_params = passed_params.pop("drop_params") special_params = passed_params.pop("kwargs") @@ -2490,6 +2494,7 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + optional_params = add_provider_specific_params_to_optional_params( optional_params=optional_params, passed_params=passed_params, @@ -4087,6 +4092,7 @@ def add_provider_specific_params_to_optional_params( """ Add provider specific params to optional_params """ + if ( custom_llm_provider in ["openai", "azure", "text-completion-openai"] @@ -7213,6 +7219,12 @@ class ProviderConfigManager: return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() + elif litellm.LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.transcriptions.transformation import ( + HostedVLLMAudioTranscriptionConfig, + ) + + return HostedVLLMAudioTranscriptionConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 289b895318..24defc6a0a 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -1,6 +1,7 @@ import inspect import os import sys +from typing import cast import pytest from pydantic import BaseModel @@ -10,6 +11,7 @@ sys.path.insert( ) from litellm.llms.ollama.chat.transformation import OllamaChatConfig +from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_optional_params @@ -101,3 +103,228 @@ class TestOllamaChatConfigResponseFormat: # Clean up class attributes delattr(litellm.OllamaChatConfig, "num_ctx") delattr(litellm.OllamaChatConfig, "temperature") + + def test_transform_request_content_list_to_string(self): + """Test that content list is properly converted to string in transform_request""" + config = OllamaChatConfig() + + # Test message with content as list containing text + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello "}, + {"type": "text", "text": "world!"}, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify content was converted to string + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "Hello world!" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_content_string_passthrough(self): + """Test that string content passes through unchanged in transform_request""" + config = OllamaChatConfig() + + # Test message with content as string + messages = cast( + list[AllMessageValues], [{"role": "user", "content": "Hello world!"}] + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify string content passes through + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "Hello world!" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_empty_content_list(self): + """Test handling of empty content list in transform_request""" + config = OllamaChatConfig() + + # Test message with empty content list + messages = cast(list[AllMessageValues], [{"role": "user", "content": []}]) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify empty content becomes empty string + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_image_extraction(self): + """Test that images are properly extracted from messages in transform_request""" + config = OllamaChatConfig() + + # Test message with images in content list + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + }, + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify text content was extracted + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "What's in this image?" + assert result["messages"][0]["role"] == "user" + + # Verify image was extracted to images list + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 1 + assert ( + result["messages"][0]["images"][0] + == "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + ) + + def test_transform_request_multiple_images_extraction(self): + """Test extraction of multiple images from a single message""" + config = OllamaChatConfig() + + # Test message with multiple images + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these images:"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,image1data..." + }, + }, + {"type": "text", "text": " and "}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,image2data..."}, + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify text content was combined + assert result["messages"][0]["content"] == "Compare these images: and " + + # Verify both images were extracted + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 2 + assert ( + result["messages"][0]["images"][0] == "data:image/jpeg;base64,image1data..." + ) + assert ( + result["messages"][0]["images"][1] == "data:image/png;base64,image2data..." + ) + + def test_transform_request_image_url_as_string(self): + """Test handling of image_url as direct string (edge case)""" + config = OllamaChatConfig() + + # Test message with image_url as string (edge case from extract_images_from_message) + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Check this:"}, + { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify image URL was extracted + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 1 + assert result["messages"][0]["images"][0] == "https://example.com/image.jpg" + + def test_transform_request_no_images_no_images_key(self): + """Test that messages without images don't have images key""" + config = OllamaChatConfig() + + # Test message with no images + messages = cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "Just text here"}]}], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify no images key when no images present + assert result["messages"][0]["content"] == "Just text here" + # Since extract_images_from_message returns empty list [] when no images found, + # and the code checks "if images is not None", an empty list will still be set + assert "images" in result["messages"][0] + assert result["messages"][0]["images"] == []