From 1bbbacea00290266e1c761457deef114c8bc3520 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 15:50:29 -0700 Subject: [PATCH 1/9] fix(key_management_endpoints.py): correctly raise an error when tags set on `/key/update` by non-premium user Closes https://github.com/BerriAI/litellm/issues/14366 --- .../index.html} | 0 .../proxy/_experimental/out/onboarding.html | 1 - .../management_endpoints/common_utils.py | 2 +- .../key_management_endpoints.py | 20 ++++++++++++++++--- 4 files changed, 18 insertions(+), 5 deletions(-) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index a9e6e893b0..0000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 5bdddfc4c6..8f2ba37a9d 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -93,7 +93,7 @@ async def _upsert_budget_and_membership( create_data["tpm_limit"] = tpm_limit if rpm_limit is not None: create_data["rpm_limit"] = rpm_limit - + new_budget = await tx.litellm_budgettable.create( data=create_data, include={"team_membership": True}, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index bd8faf34be..7538d22453 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -346,6 +346,7 @@ def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict: data_json["allowed_routes"] = ["info_routes"] return data_json + async def validate_team_id_used_in_service_account_request( team_id: Optional[str], prisma_client: Optional[PrismaClient], @@ -358,13 +359,13 @@ async def validate_team_id_used_in_service_account_request( status_code=400, detail="team_id is required for service account keys. Please specify `team_id` in the request body.", ) - + if prisma_client is None: raise HTTPException( status_code=400, detail="prisma_client is required for service account keys. Please specify `prisma_client` in the request body.", ) - + # check if team_id exists in the database team = await prisma_client.db.litellm_teamtable.find_unique( where={"team_id": team_id}, @@ -376,6 +377,7 @@ async def validate_team_id_used_in_service_account_request( ) return True + async def _common_key_generation_helper( # noqa: PLR0915 data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -557,7 +559,7 @@ async def _common_key_generation_helper( # noqa: PLR0915 status_code=400, detail={ "error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {data.key}" - } + }, ) response = await generate_key_helper_fn( @@ -923,6 +925,15 @@ async def prepare_key_update_data( detail="team_id is required for service account keys. Please specify `team_id` in the request body.", ) non_default_values = {} + # ADD METADATA FIELDS + # Set Management Endpoint Metadata Fields + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) for k, v in data_json.items(): if ( k in LiteLLM_ManagementEndpoint_MetadataFields @@ -1135,6 +1146,9 @@ async def update_key_fn( change_initiated_by=user_api_key_dict, llm_router=llm_router, ) + + # Set Management Endpoint Metadata Fields + non_default_values = await prepare_key_update_data( data=data, existing_key_row=existing_key_row ) From 82091de39349c96d3477641ad5f47e4eae0cd64b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 17:15:14 -0700 Subject: [PATCH 2/9] feat(hosted_vllm/): transcription endpoint support Closes https://github.com/BerriAI/litellm/issues/361#issuecomment-3244548055 --- litellm/constants.py | 10 ++- .../audio_transcription/transformation.py | 10 +-- litellm/llms/custom_httpx/llm_http_handler.py | 52 +++++++---- .../transcriptions/transformation.py | 86 +++++++++++++++++++ .../transcriptions/gpt_transformation.py | 11 ++- litellm/llms/openai/transcriptions/handler.py | 7 +- .../transcriptions/whisper_transformation.py | 39 +++++++-- litellm/main.py | 54 +++++++----- litellm/proxy/_new_secret_config.yaml | 5 ++ litellm/utils.py | 18 +++- 10 files changed, 229 insertions(+), 63 deletions(-) create mode 100644 litellm/llms/hosted_vllm/transcriptions/transformation.py diff --git a/litellm/constants.py b/litellm/constants.py index 75c25d9ea9..c0ce0f265b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) DEFAULT_NUM_WORKERS_LITELLM_PROXY = int( - os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4) + os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) ) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" @@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int( os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128) ) DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512) + os.getenv( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512 + ) ) # Generic fallback for unknown models @@ -949,7 +951,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token" DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics" CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data" -CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) +CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int( + os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000) +) SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup" SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 179b8d0fb0..c20ee0f737 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -23,12 +23,13 @@ else: class AudioTranscriptionRequestData: """ Structured data for audio transcription requests. - + Attributes: data: The request data (form data for multipart, json data for regular requests) files: Optional files dict for multipart form data content_type: Optional content type override """ + data: Union[dict, bytes] files: Optional[dict] = None content_type: Optional[str] = None @@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> Union[AudioTranscriptionRequestData, Dict]: + ) -> AudioTranscriptionRequestData: raise NotImplementedError( "AudioTranscriptionConfig needs a request transformation for audio transcription models" ) - - def transform_audio_transcription_response( self, raw_response: httpx.Response, @@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): raise NotImplementedError( "AudioTranscriptionConfig does not need a response transformation for audio transcription models" ) - def get_provider_specific_params( self, @@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): provider_specific_params[key] = value return provider_specific_params - + def _should_exclude_param( self, param_name: str, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 13133a56aa..5dc0d2bb95 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2221,7 +2221,9 @@ class BaseLLMHTTPHandler: if isinstance(transformed_request, dict) and "method" in transformed_request: # Handle pre-signed requests (e.g., from Bedrock S3 uploads) - upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())( + upload_response = getattr( + sync_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2233,8 +2235,8 @@ class BaseLLMHTTPHandler: # Handle traditional file uploads # Ensure transformed_request is a string for httpx compatibility if isinstance(transformed_request, bytes): - transformed_request = transformed_request.decode('utf-8') - + transformed_request = transformed_request.decode("utf-8") + # Use the HTTP method specified by the provider config http_method = provider_config.file_upload_http_method.upper() if http_method == "PUT": @@ -2310,7 +2312,7 @@ class BaseLLMHTTPHandler: ) else: async_httpx_client = client - + ######################################################### # Debug Logging ######################################################### @@ -2326,7 +2328,9 @@ class BaseLLMHTTPHandler: if isinstance(transformed_request, dict) and "method" in transformed_request: # Handle pre-signed requests (e.g., from Bedrock S3 uploads) - upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())( + upload_response = await getattr( + async_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2338,8 +2342,8 @@ class BaseLLMHTTPHandler: # Handle traditional file uploads # Ensure transformed_request is a string for httpx compatibility if isinstance(transformed_request, bytes): - transformed_request = transformed_request.decode('utf-8') - + transformed_request = transformed_request.decode("utf-8") + # Use the HTTP method specified by the provider config http_method = provider_config.file_upload_http_method.upper() if http_method == "PUT": @@ -2460,9 +2464,14 @@ class BaseLLMHTTPHandler: sync_httpx_client = client try: - if isinstance(transformed_request, dict) and "method" in transformed_request: + if ( + isinstance(transformed_request, dict) + and "method" in transformed_request + ): # Handle pre-signed requests (e.g., from Bedrock with AWS auth) - batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())( + batch_response = getattr( + sync_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2492,8 +2501,11 @@ class BaseLLMHTTPHandler: ) # Store original request for response transformation - litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data} - + litellm_params_with_request = { + **litellm_params, + "original_batch_request": create_batch_data, + } + return provider_config.transform_create_batch_response( model=None, raw_response=batch_response, @@ -2522,7 +2534,7 @@ class BaseLLMHTTPHandler: ) else: async_httpx_client = client - + ######################################################### # Debug Logging ######################################################### @@ -2537,9 +2549,14 @@ class BaseLLMHTTPHandler: ) try: - if isinstance(transformed_request, dict) and "method" in transformed_request: + if ( + isinstance(transformed_request, dict) + and "method" in transformed_request + ): # Handle pre-signed requests (e.g., from Bedrock with AWS auth) - batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())( + batch_response = await getattr( + async_httpx_client, transformed_request["method"].lower() + )( url=transformed_request["url"], headers=transformed_request["headers"], data=transformed_request["data"], @@ -2569,8 +2586,11 @@ class BaseLLMHTTPHandler: ) # Store original request for response transformation (for async version) - litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}} - + litellm_params_with_request = { + **litellm_params, + "original_batch_request": create_batch_data or {}, + } + return provider_config.transform_create_batch_response( model=None, raw_response=batch_response, diff --git a/litellm/llms/hosted_vllm/transcriptions/transformation.py b/litellm/llms/hosted_vllm/transcriptions/transformation.py new file mode 100644 index 0000000000..14ed278c97 --- /dev/null +++ b/litellm/llms/hosted_vllm/transcriptions/transformation.py @@ -0,0 +1,86 @@ +""" +Transformation logic for Hosted VLLM rerank +""" + +import uuid +from typing import Any, Dict, List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.openai.transcriptions.whisper_transformation import ( + OpenAIWhisperAudioTranscriptionConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.rerank import ( + OptionalRerankParams, + RerankBilledUnits, + RerankRequest, + RerankResponse, + RerankResponseDocument, + RerankResponseMeta, + RerankResponseResult, + RerankTokens, +) +from litellm.types.utils import FileTypes + + +class HostedVLLMAudioTranscriptionError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: Optional[Union[dict, httpx.Headers]] = None, + ): + super().__init__(status_code=status_code, message=message, headers=headers) + + +class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): + def __init__(self) -> None: + pass + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base: + # Remove trailing slashes and ensure clean base URL + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/audio/transcriptions"): + api_base = f"{api_base}/v1/audio/transcriptions" + return api_base + raise ValueError("api_base must be provided for Hosted VLLM rerank") + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + """ + Transform the audio transcription request + """ + + data = {"model": model, "file": audio_file, **optional_params} + + if "response_format" not in data or ( + data["response_format"] == "text" or data["response_format"] == "json" + ): + data["response_format"] = ( + "verbose_json" # ensures 'duration' is received - used for cost calculation + ) + + return AudioTranscriptionRequestData( + data=data, + ) diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index 796e10f515..34621c44e2 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -1,5 +1,8 @@ from typing import List +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams from litellm.types.utils import FileTypes @@ -27,8 +30,12 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> dict: + ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request """ - return {"model": model, "file": audio_file, **optional_params} + data = {"model": model, "file": audio_file, **optional_params} + + return AudioTranscriptionRequestData( + data=data, + ) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 4fe48dd3c6..03ac34f3dd 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import Optional, Union, cast import httpx from openai import AsyncOpenAI, OpenAI @@ -93,15 +93,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): Handle audio transcription request """ if provider_config is not None: - data = provider_config.transform_audio_transcription_request( + transformed_data = provider_config.transform_audio_transcription_request( model=model, audio_file=audio_file, optional_params=optional_params, litellm_params=litellm_params, ) - if not isinstance(data, dict): - raise ValueError("OpenAI transformation route requires a dict") + data = cast(dict, transformed_data.data) else: data = {"model": model, "file": audio_file, **optional_params} diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index c0ccc71579..bb14b4d47a 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -1,8 +1,9 @@ from typing import List, Optional, Union -from httpx import Headers +from httpx import Headers, Response from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -11,7 +12,7 @@ from litellm.types.llms.openai import ( AllMessageValues, OpenAIAudioTranscriptionOptionalParams, ) -from litellm.types.utils import FileTypes +from litellm.types.utils import FileTypes, TranscriptionResponse from ..common_utils import OpenAIError @@ -72,7 +73,7 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): audio_file: FileTypes, optional_params: dict, litellm_params: dict, - ) -> dict: + ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request """ @@ -82,11 +83,13 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if "response_format" not in data or ( data["response_format"] == "text" or data["response_format"] == "json" ): - data[ - "response_format" - ] = "verbose_json" # ensures 'duration' is received - used for cost calculation + data["response_format"] = ( + "verbose_json" # ensures 'duration' is received - used for cost calculation + ) - return data + return AudioTranscriptionRequestData( + data=data, + ) def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, Headers] @@ -96,3 +99,25 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): message=error_message, headers=headers, ) + + def transform_audio_transcription_response( + self, + raw_response: Response, + ) -> TranscriptionResponse: + try: + raw_response_json = raw_response.json() + except Exception as e: + raise ValueError( + f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}" + ) + + if any( + key in raw_response_json + for key in TranscriptionResponse.model_fields.keys() + ): + return TranscriptionResponse(**raw_response_json) + else: + raise ValueError( + "Invalid response format. Received response does not match the expected format. Got: ", + raw_response_json, + ) diff --git a/litellm/main.py b/litellm/main.py index d7395eb145..b74f7e1d0f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm +from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig -from .llms.gemini.common_utils import get_api_key_from_env from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.oci.chat.transformation import OCIChatConfig @@ -358,7 +358,9 @@ async def acompletion( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, deployment_id=None, - reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None, + reasoning_effort: Optional[ + Literal["none", "minimal", "low", "medium", "high", "default"] + ] = None, safety_identifier: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, @@ -504,7 +506,9 @@ async def acompletion( } if custom_llm_provider is None: _, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None) + model=model, + custom_llm_provider=custom_llm_provider, + api_base=completion_kwargs.get("base_url", None), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -899,7 +903,9 @@ def completion( # type: ignore # noqa: PLR0915 logit_bias: Optional[dict] = None, user: Optional[str] = None, # openai v1.0+ new params - reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None, + reasoning_effort: Optional[ + Literal["none", "minimal", "low", "medium", "high", "default"] + ] = None, response_format: Optional[Union[dict, Type[BaseModel]]] = None, seed: Optional[int] = None, tools: Optional[List] = None, @@ -1116,10 +1122,12 @@ def completion( # type: ignore # noqa: PLR0915 ) if provider_specific_header is not None: - headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers( - provider_specific_header=provider_specific_header, - custom_llm_provider=custom_llm_provider, - )) + headers.update( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, + custom_llm_provider=custom_llm_provider, + ) + ) if model_response is not None and hasattr(model_response, "_hidden_params"): model_response._hidden_params["custom_llm_provider"] = custom_llm_provider @@ -2712,9 +2720,7 @@ def completion( # type: ignore # noqa: PLR0915 ) api_key = ( - api_key - or litellm.api_key - or get_secret("VERCEL_AI_GATEWAY_API_KEY") + api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") ) vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" @@ -2730,7 +2736,7 @@ def completion( # type: ignore # noqa: PLR0915 vercel_headers.update(_headers) headers = vercel_headers - + ## Load Config config = litellm.VercelAIGatewayConfig.get_config() for k, v in config.items(): @@ -3712,7 +3718,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None) + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), ) # Await normally @@ -5338,9 +5346,9 @@ def transcription( max_retries=max_retries, litellm_params=litellm_params_dict, ) - elif ( - custom_llm_provider == "openai" - or custom_llm_provider in litellm.openai_compatible_providers + elif custom_llm_provider == "openai" or ( + custom_llm_provider in litellm.openai_compatible_providers + and provider_config is None ): api_base = ( api_base @@ -5371,10 +5379,7 @@ def transcription( provider_config=provider_config, litellm_params=litellm_params_dict, ) - elif custom_llm_provider in [ - LlmProviders.DEEPGRAM.value, - LlmProviders.ELEVENLABS.value, - ]: + elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( model=model, audio_file=file, @@ -5780,7 +5785,14 @@ async def ahealth_check( input=input or ["test"], ), "audio_speech": lambda: litellm.aspeech( - **{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})}, + **{ + **_filter_model_params(model_params), + **( + {"voice": "alloy"} + if "voice" not in _filter_model_params(model_params) + else {} + ), + }, input=prompt or "test", ), "audio_transcription": lambda: litellm.atranscription( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index c785dd05c4..e93902039b 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -7,3 +7,8 @@ model_list: - model_name: wildcard_models/* litellm_params: model: openai/* + - model_name: hosted_vllm/* + litellm_params: + model: hosted_vllm/* + api_base: https://webhook.site/6fbe498e-88b5-4a5f-8f07-edb9806c1937 + api_key: fake-key diff --git a/litellm/utils.py b/litellm/utils.py index 0d2fe5d4d6..a20bd904d6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2437,7 +2437,7 @@ def get_optional_params_transcription( "prompt": None, "response_format": None, "temperature": None, # openai defaults this to 0 - "timestamp_granularities": None + "timestamp_granularities": None, } non_default_params = { @@ -2505,7 +2505,7 @@ def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str: """Map OpenAI size parameter to Vertex AI aspectRatio.""" if size is None: return "1:1" - + # Map OpenAI size strings to Vertex AI aspect ratio strings # Vertex AI accepts: "1:1", "9:16", "16:9", "4:3", "3:4" size_to_aspect_ratio = { @@ -2515,7 +2515,9 @@ def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str: "1792x1024": "16:9", # Landscape "1024x1792": "9:16", # Portrait } - return size_to_aspect_ratio.get(size, "1:1") # Default to square if size not recognized + return size_to_aspect_ratio.get( + size, "1:1" + ) # Default to square if size not recognized def get_optional_params_image_gen( @@ -2631,7 +2633,9 @@ def get_optional_params_image_gen( # Map OpenAI size parameter to Vertex AI aspectRatio if size is not None: - optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(size) + optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio( + size + ) openai_params: list[str] = list(default_params.keys()) if provider_config is not None: @@ -7209,6 +7213,12 @@ class ProviderConfigManager: return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() + elif litellm.LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.transcriptions.transformation import ( + HostedVLLMAudioTranscriptionConfig, + ) + + return HostedVLLMAudioTranscriptionConfig() return None @staticmethod From dd663f80ce151535e236a9f62edf41774f492ba3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 17:41:25 -0700 Subject: [PATCH 3/9] feat(ollama/chat): ensure content is str - even when input is list[str] Fixes https://github.com/BerriAI/litellm/issues/14217 --- .../convert_dict_to_response.py | 47 ++----------------- .../prompt_templates/common_utils.py | 44 +++++++++++++++++ .../bedrock/chat/converse_transformation.py | 18 ++++--- .../amazon_deepseek_transformation.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 1 - litellm/llms/ollama/chat/transformation.py | 27 +++++++++-- .../llms/ollama/completion/transformation.py | 4 +- litellm/proxy/_new_secret_config.yaml | 3 ++ litellm/types/llms/ollama.py | 9 ++++ 9 files changed, 98 insertions(+), 57 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8dc3061460..2c5d930d58 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -9,6 +9,10 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union import litellm from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, + _parse_content_for_reasoning, +) from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, @@ -274,49 +278,6 @@ def _handle_invalid_parallel_tool_calls( return tool_calls -def _parse_content_for_reasoning( - message_text: Optional[str], -) -> Tuple[Optional[str], Optional[str]]: - """ - Parse the content for reasoning - - Returns: - - reasoning_content: The content of the reasoning - - content: The content of the message - """ - if not message_text: - return None, message_text - - reasoning_match = re.match( - r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL - ) - - if reasoning_match: - return reasoning_match.group(1), reasoning_match.group(2) - - return None, message_text - - -def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: - """ - Extract reasoning content and main content from a message. - - Args: - message (dict): The message dictionary that may contain reasoning_content - - Returns: - tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) - """ - message_content = message.get("content") - if "reasoning_content" in message: - return message["reasoning_content"], message["content"] - elif "reasoning" in message: - return message["reasoning"], message["content"] - elif isinstance(message_content, str): - return _parse_content_for_reasoning(message_content) - return None, message_content - - class LiteLLMResponseObjectHandler: @staticmethod def convert_to_image_response( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a99883ef7b..c5aa230427 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -14,6 +14,7 @@ from typing import ( Literal, Mapping, Optional, + Tuple, Union, cast, ) @@ -869,3 +870,46 @@ def convert_prefix_message_to_non_prefix_messages( else: new_messages.append(message) return new_messages + + +def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: + """ + Extract reasoning content and main content from a message. + + Args: + message (dict): The message dictionary that may contain reasoning_content + + Returns: + tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) + """ + message_content = message.get("content") + if "reasoning_content" in message: + return message["reasoning_content"], message["content"] + elif "reasoning" in message: + return message["reasoning"], message["content"] + elif isinstance(message_content, str): + return _parse_content_for_reasoning(message_content) + return None, message_content + + +def _parse_content_for_reasoning( + message_text: Optional[str], +) -> Tuple[Optional[str], Optional[str]]: + """ + Parse the content for reasoning + + Returns: + - reasoning_content: The content of the reasoning + - content: The content of the message + """ + if not message_text: + return None, message_text + + reasoning_match = re.match( + r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL + ) + + if reasoning_match: + return reasoning_match.group(1), reasoning_match.group(2) + + return None, message_text diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fda9220ff7..e3d65be8bb 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -14,7 +14,7 @@ from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -397,7 +397,11 @@ class AmazonConverseConfig(BaseConfig): for param, value in non_default_params.items(): if param == "response_format" and isinstance(value, dict): optional_params = self._translate_response_format_param( - value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled + value=value, + model=model, + optional_params=optional_params, + non_default_params=non_default_params, + is_thinking_enabled=is_thinking_enabled, ) if param == "max_tokens" or param == "max_completion_tokens": optional_params["maxTokens"] = value @@ -446,11 +450,11 @@ class AmazonConverseConfig(BaseConfig): ) return optional_params - + def _translate_response_format_param( - self, - value: dict, - model: str, + self, + value: dict, + model: str, optional_params: dict, non_default_params: dict, is_thinking_enabled: bool, @@ -504,7 +508,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["json_mode"] = True if non_default_params.get("stream", False) is True: optional_params["fake_stream"] = True - + return optional_params def update_optional_params_with_thinking_tokens( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index d7ceec1f1c..0fe84b0ce0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast from httpx import Response from litellm import verbose_logger -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5dc0d2bb95..f871770615 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -118,7 +118,6 @@ class BaseLLMHTTPHandler: response: Optional[httpx.Response] = None for i in range(max(max_retry_on_unprocessable_entity_error, 1)): try: - response = await async_httpx_client.post( url=api_base, headers=headers, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index ee0d3acef7..de61ae6e3b 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,9 +16,17 @@ from httpx._models import Headers, Response from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, + convert_content_list_to_str, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction +from litellm.types.llms.ollama import ( + OllamaChatCompletionMessage, + OllamaToolCall, + OllamaToolCallFunction, +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantToolCall, @@ -299,7 +307,20 @@ class OllamaChatConfig(BaseConfig): ) new_tools.append(ollama_tool_call) cast(dict, m)["tool_calls"] = new_tools - new_messages.append(m) + reasoning_content, parsed_content = _extract_reasoning_content( + cast(dict, m) + ) + content_str = convert_content_list_to_str(cast(AllMessageValues, m)) + + ollama_message = OllamaChatCompletionMessage( + role=cast(str, m.get("role")), + ) + if reasoning_content is not None: + ollama_message["thinking"] = reasoning_content + if content_str is not None: + ollama_message["content"] = content_str + + new_messages.append(ollama_message) # Load Config config = self.get_config() @@ -361,7 +382,7 @@ class OllamaChatConfig(BaseConfig): del response_json_message["thinking"] elif response_json_message.get("content") is not None: # parse reasoning content from content - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 71bcf0bb3f..bfb0b7f187 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -229,7 +229,7 @@ class OllamaConfig(BaseConfig): model = model.split("/", 1)[1] api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" api_key = self.get_api_key() - headers = { "Authorization": f"Bearer {api_key}" } if api_key else {} + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} try: response = litellm.module_level_client.post( @@ -279,7 +279,7 @@ class OllamaConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index e93902039b..c30ac5f44a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -12,3 +12,6 @@ model_list: model: hosted_vllm/* api_base: https://webhook.site/6fbe498e-88b5-4a5f-8f07-edb9806c1937 api_key: fake-key + - model_name: deepseek-r1-5b + litellm_params: + model: ollama_chat/deepseek-r1:1.5b diff --git a/litellm/types/llms/ollama.py b/litellm/types/llms/ollama.py index 9d71904caa..d8f20de514 100644 --- a/litellm/types/llms/ollama.py +++ b/litellm/types/llms/ollama.py @@ -27,3 +27,12 @@ class OllamaToolCall(TypedDict): class OllamaVisionModelObject(TypedDict): prompt: str images: List[str] + + +class OllamaChatCompletionMessage(TypedDict, total=False): + role: Required[str] + content: str + thinking: str + images: List[str] + tool_calls: List[OllamaToolCall] + tool_name: str From 461e18145098ca2c6a7f27568c0ffb49a9d1d126 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 17:49:42 -0700 Subject: [PATCH 4/9] fix(ollama/chat): support images Fixes https://github.com/BerriAI/litellm/issues/14217 --- .../prompt_templates/common_utils.py | 17 +++++++++++++++++ litellm/llms/ollama/chat/transformation.py | 4 ++++ litellm/router.py | 16 ++++++++++------ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index c5aa230427..19d5932ff2 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -913,3 +913,20 @@ def _parse_content_for_reasoning( return reasoning_match.group(1), reasoning_match.group(2) return None, message_text + + +def extract_images_from_message(message: AllMessageValues) -> List[str]: + """ + Extract images from a message + """ + images = [] + message_content = message.get("content") + if isinstance(message_content, list): + for m in message_content: + image_url = m.get("image_url") + if image_url: + if isinstance(image_url, str): + images.append(image_url) + elif isinstance(image_url, dict) and "url" in image_url: + images.append(image_url["url"]) + return images diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index de61ae6e3b..3527a57921 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -19,6 +19,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( _extract_reasoning_content, convert_content_list_to_str, + extract_images_from_message, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -311,6 +312,7 @@ class OllamaChatConfig(BaseConfig): cast(dict, m) ) content_str = convert_content_list_to_str(cast(AllMessageValues, m)) + images = extract_images_from_message(cast(AllMessageValues, m)) ollama_message = OllamaChatCompletionMessage( role=cast(str, m.get("role")), @@ -319,6 +321,8 @@ class OllamaChatConfig(BaseConfig): ollama_message["thinking"] = reasoning_content if content_str is not None: ollama_message["content"] = content_str + if images is not None: + ollama_message["images"] = images new_messages.append(ollama_message) diff --git a/litellm/router.py b/litellm/router.py index 1491adf4df..b5d10237ee 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4414,7 +4414,7 @@ class Router: return tpm_key except Exception as e: - verbose_router_logger.exception( + verbose_router_logger.debug( "litellm.router.Router::deployment_callback_on_success(): Exception occured - {}".format( str(e) ) @@ -4562,8 +4562,10 @@ class Router: parent_otel_span=parent_otel_span, ttl=RoutingArgs.ttl.value, ) - - def _get_metadata_variable_name_from_kwargs(self, kwargs: dict) -> Literal["metadata", "litellm_metadata"]: + + def _get_metadata_variable_name_from_kwargs( + self, kwargs: dict + ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data @@ -5672,11 +5674,11 @@ class Router: ) if supported_openai_params is None: supported_openai_params = [] - + # Get mode from database model_info if available, otherwise default to "chat" db_model_info = model.get("model_info", {}) mode = db_model_info.get("mode", "chat") - + model_info = ModelMapInfo( key=model_group, max_tokens=None, @@ -6802,7 +6804,9 @@ class Router: model=model, request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, - metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + metadata_variable_name=self._get_metadata_variable_name_from_kwargs( + request_kwargs + ), ) if len(healthy_deployments) == 0: From 8ec02a5bdcaf8f5d529fa1f1dd86a9f24313afd0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 17:59:59 -0700 Subject: [PATCH 5/9] test: add unit tests for ollama chat request transformation --- .../ollama/test_ollama_chat_transformation.py | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 289b895318..24defc6a0a 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -1,6 +1,7 @@ import inspect import os import sys +from typing import cast import pytest from pydantic import BaseModel @@ -10,6 +11,7 @@ sys.path.insert( ) from litellm.llms.ollama.chat.transformation import OllamaChatConfig +from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_optional_params @@ -101,3 +103,228 @@ class TestOllamaChatConfigResponseFormat: # Clean up class attributes delattr(litellm.OllamaChatConfig, "num_ctx") delattr(litellm.OllamaChatConfig, "temperature") + + def test_transform_request_content_list_to_string(self): + """Test that content list is properly converted to string in transform_request""" + config = OllamaChatConfig() + + # Test message with content as list containing text + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello "}, + {"type": "text", "text": "world!"}, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify content was converted to string + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "Hello world!" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_content_string_passthrough(self): + """Test that string content passes through unchanged in transform_request""" + config = OllamaChatConfig() + + # Test message with content as string + messages = cast( + list[AllMessageValues], [{"role": "user", "content": "Hello world!"}] + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify string content passes through + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "Hello world!" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_empty_content_list(self): + """Test handling of empty content list in transform_request""" + config = OllamaChatConfig() + + # Test message with empty content list + messages = cast(list[AllMessageValues], [{"role": "user", "content": []}]) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify empty content becomes empty string + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "" + assert result["messages"][0]["role"] == "user" + + def test_transform_request_image_extraction(self): + """Test that images are properly extracted from messages in transform_request""" + config = OllamaChatConfig() + + # Test message with images in content list + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + }, + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify text content was extracted + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "What's in this image?" + assert result["messages"][0]["role"] == "user" + + # Verify image was extracted to images list + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 1 + assert ( + result["messages"][0]["images"][0] + == "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + ) + + def test_transform_request_multiple_images_extraction(self): + """Test extraction of multiple images from a single message""" + config = OllamaChatConfig() + + # Test message with multiple images + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these images:"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,image1data..." + }, + }, + {"type": "text", "text": " and "}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,image2data..."}, + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify text content was combined + assert result["messages"][0]["content"] == "Compare these images: and " + + # Verify both images were extracted + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 2 + assert ( + result["messages"][0]["images"][0] == "data:image/jpeg;base64,image1data..." + ) + assert ( + result["messages"][0]["images"][1] == "data:image/png;base64,image2data..." + ) + + def test_transform_request_image_url_as_string(self): + """Test handling of image_url as direct string (edge case)""" + config = OllamaChatConfig() + + # Test message with image_url as string (edge case from extract_images_from_message) + messages = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Check this:"}, + { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + }, + ], + } + ], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify image URL was extracted + assert "images" in result["messages"][0] + assert len(result["messages"][0]["images"]) == 1 + assert result["messages"][0]["images"][0] == "https://example.com/image.jpg" + + def test_transform_request_no_images_no_images_key(self): + """Test that messages without images don't have images key""" + config = OllamaChatConfig() + + # Test message with no images + messages = cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "Just text here"}]}], + ) + + result = config.transform_request( + model="llama2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # Verify no images key when no images present + assert result["messages"][0]["content"] == "Just text here" + # Since extract_images_from_message returns empty list [] when no images found, + # and the code checks "if images is not None", an empty list will still be set + assert "images" in result["messages"][0] + assert result["messages"][0]["images"] == [] From 7a5e5a12da77dd1182716ba3a9448f9974db939f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 12 Sep 2025 18:03:50 -0700 Subject: [PATCH 6/9] fix: fix linting errors --- .../convert_dict_to_response.py | 2 -- .../audio_transcription/transformation.py | 2 +- .../hosted_vllm/transcriptions/transformation.py | 16 +--------------- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 2c5d930d58..ce054b91cc 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1,6 +1,5 @@ import asyncio import json -import re import time import traceback import uuid @@ -11,7 +10,6 @@ from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.common_utils import ( _extract_reasoning_content, - _parse_content_for_reasoning, ) from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index c20ee0f737..3574996e48 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx diff --git a/litellm/llms/hosted_vllm/transcriptions/transformation.py b/litellm/llms/hosted_vllm/transcriptions/transformation.py index 14ed278c97..5eeb892d84 100644 --- a/litellm/llms/hosted_vllm/transcriptions/transformation.py +++ b/litellm/llms/hosted_vllm/transcriptions/transformation.py @@ -2,31 +2,17 @@ Transformation logic for Hosted VLLM rerank """ -import uuid -from typing import Any, Dict, List, Optional, Union +from typing import Optional, Union import httpx -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.openai.transcriptions.whisper_transformation import ( OpenAIWhisperAudioTranscriptionConfig, ) -from litellm.secret_managers.main import get_secret_str -from litellm.types.rerank import ( - OptionalRerankParams, - RerankBilledUnits, - RerankRequest, - RerankResponse, - RerankResponseDocument, - RerankResponseMeta, - RerankResponseResult, - RerankTokens, -) from litellm.types.utils import FileTypes From 9ee9745781f2c8b00d3a86bd55affa24be037baa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 13 Sep 2025 09:46:39 -0700 Subject: [PATCH 7/9] docs(vllm.md): document new endpoint --- docs/my-website/docs/providers/vllm.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/vllm.md b/docs/my-website/docs/providers/vllm.md index 5472f0602f..1a37f2f10e 100644 --- a/docs/my-website/docs/providers/vllm.md +++ b/docs/my-website/docs/providers/vllm.md @@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM. | Property | Details | |-------|-------| | Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) | -| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) | +| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) | | Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) | -| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank` | +| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` | # Quick Start From a8e2d24d3a3da31c273b32da0a0e2fd5d6a5b3ff Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 13 Sep 2025 10:12:32 -0700 Subject: [PATCH 8/9] fix: fix import --- litellm/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index a20bd904d6..423f3cfa79 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -107,7 +107,6 @@ from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_s from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( LiteLLMResponseObjectHandler, _handle_invalid_parallel_tool_calls, - _parse_content_for_reasoning, convert_to_model_response_object, convert_to_streaming_response, convert_to_streaming_response_async, @@ -122,6 +121,9 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ResponseMetadata, ) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, +) from litellm.litellm_core_utils.redact_messages import ( LiteLLMLoggingObject, redact_message_input_output_from_logging, From 8443000ca4551ff3108a40e65674191317f111b8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 13 Sep 2025 11:49:05 -0700 Subject: [PATCH 9/9] fix(main.py): route vllm calls via the openai sdk route consistent with other openai-like implementations --- litellm/llms/openai/transcriptions/handler.py | 1 + .../transcriptions/whisper_transformation.py | 29 ++++++++++++++++++- litellm/main.py | 8 +++-- litellm/proxy/_new_secret_config.yaml | 8 +++++ litellm/utils.py | 4 +++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 03ac34f3dd..19b303bb96 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -34,6 +34,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: + raw_response = ( await openai_aclient.audio.transcriptions.with_raw_response.create( **data, timeout=timeout diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index bb14b4d47a..fa507e1bc2 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -18,6 +18,34 @@ from ..common_utils import OpenAIError class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + ## get the api base, attach the endpoint - v1/audio/transcriptions + # strip trailing slash if present + api_base = api_base.rstrip("/") if api_base else "" + + # if endswith "/v1" + if api_base and api_base.endswith("/v1"): + api_base = f"{api_base}/audio/transcriptions" + else: + api_base = f"{api_base}/v1/audio/transcriptions" + + return api_base or "" + def get_supported_openai_params( self, model: str ) -> List[OpenAIAudioTranscriptionOptionalParams]: @@ -77,7 +105,6 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): """ Transform the audio transcription request """ - data = {"model": model, "file": audio_file, **optional_params} if "response_format" not in data or ( diff --git a/litellm/main.py b/litellm/main.py index 27ffaeba79..c910c88d17 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5267,7 +5267,10 @@ def transcription( model_response = litellm.utils.TranscriptionResponse() model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, ) # type: ignore if dynamic_api_key is not None: @@ -5283,6 +5286,7 @@ def transcription( custom_llm_provider=custom_llm_provider, **non_default_params, ) + litellm_params_dict = get_litellm_params(**kwargs) litellm_logging_obj.update_environment_variables( @@ -5349,7 +5353,6 @@ def transcription( ) elif custom_llm_provider == "openai" or ( custom_llm_provider in litellm.openai_compatible_providers - and provider_config is None ): api_base = ( api_base @@ -5364,6 +5367,7 @@ def transcription( or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) # set API KEY + api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") # type: ignore response = openai_audio_transcriptions.audio_transcriptions( model=model, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 7ed87144c0..8b9f81b41a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -10,3 +10,11 @@ model_list: - model_name: xai-grok-3 litellm_params: model: xai/grok-3 + - model_name: hosted_vllm/whisper-v3 + litellm_params: + model: hosted_vllm/whisper-v3 + api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" + api_key: dummy + + + diff --git a/litellm/utils.py b/litellm/utils.py index 423f3cfa79..31b94984e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2428,6 +2428,8 @@ def get_optional_params_transcription( # retrieve all parameters passed to the function passed_params = locals() + + passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") custom_llm_provider = passed_params.pop("custom_llm_provider") drop_params = passed_params.pop("drop_params") special_params = passed_params.pop("kwargs") @@ -2492,6 +2494,7 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + optional_params = add_provider_specific_params_to_optional_params( optional_params=optional_params, passed_params=passed_params, @@ -4089,6 +4092,7 @@ def add_provider_specific_params_to_optional_params( """ Add provider specific params to optional_params """ + if ( custom_llm_provider in ["openai", "azure", "text-completion-openai"]