tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-21 08:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14523 from BerriAI/litellm_dev_09_12_2025_p1

Browse Source 
VLLM - transcription endpoint support + Ollama_chat/ - images, thinking, and content as list handling +
This commit is contained in:
Krish Dholakia
2025-09-13 13:39:28 -07:00
committed by GitHub
parent 6c27e5ce43 8443000ca4
commit 6efc898407
21 changed files with 584 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/providers/vllm.md
+2 -2
View File
@@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM.
| Property | Details |
|-------|-------|
| Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) |
| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) |
| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) |
| Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) |
| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank` |
| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` |
# Quick Start
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
+3 -44
View File
@@ -1,6 +1,5 @@
import asyncio
import json
import re
import time
import traceback
import uuid
@@ -9,6 +8,9 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
import litellm
from litellm._logging import verbose_logger
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
)
from litellm.types.llms.databricks import DatabricksTool
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
@@ -274,49 +276,6 @@ def _handle_invalid_parallel_tool_calls(
return tool_calls
def _parse_content_for_reasoning(
message_text: Optional[str],
) -> Tuple[Optional[str], Optional[str]]:
"""
Parse the content for reasoning
Returns:
- reasoning_content: The content of the reasoning
- content: The content of the message
"""
if not message_text:
return None, message_text
reasoning_match = re.match(
r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
)
if reasoning_match:
return reasoning_match.group(1), reasoning_match.group(2)
return None, message_text
def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
"""
Extract reasoning content and main content from a message.
Args:
message (dict): The message dictionary that may contain reasoning_content
Returns:
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
"""
message_content = message.get("content")
if "reasoning_content" in message:
return message["reasoning_content"], message["content"]
elif "reasoning" in message:
return message["reasoning"], message["content"]
elif isinstance(message_content, str):
return _parse_content_for_reasoning(message_content)
return None, message_content
class LiteLLMResponseObjectHandler:
@staticmethod
def convert_to_image_response(
litellm/litellm_core_utils/prompt_templates/common_utils.py
+61
View File
@@ -14,6 +14,7 @@ from typing import (
Literal,
Mapping,
Optional,
Tuple,
Union,
cast,
)
@@ -869,3 +870,63 @@ def convert_prefix_message_to_non_prefix_messages(
else:
new_messages.append(message)
return new_messages
def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
"""
Extract reasoning content and main content from a message.
Args:
message (dict): The message dictionary that may contain reasoning_content
Returns:
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
"""
message_content = message.get("content")
if "reasoning_content" in message:
return message["reasoning_content"], message["content"]
elif "reasoning" in message:
return message["reasoning"], message["content"]
elif isinstance(message_content, str):
return _parse_content_for_reasoning(message_content)
return None, message_content
def _parse_content_for_reasoning(
message_text: Optional[str],
) -> Tuple[Optional[str], Optional[str]]:
"""
Parse the content for reasoning
Returns:
- reasoning_content: The content of the reasoning
- content: The content of the message
"""
if not message_text:
return None, message_text
reasoning_match = re.match(
r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
)
if reasoning_match:
return reasoning_match.group(1), reasoning_match.group(2)
return None, message_text
def extract_images_from_message(message: AllMessageValues) -> List[str]:
"""
Extract images from a message
"""
images = []
message_content = message.get("content")
if isinstance(message_content, list):
for m in message_content:
image_url = m.get("image_url")
if image_url:
if isinstance(image_url, str):
images.append(image_url)
elif isinstance(image_url, dict) and "url" in image_url:
images.append(image_url["url"])
return images
litellm/llms/base_llm/audio_transcription/transformation.py
+5 -7
View File
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, List, Optional, Union
import httpx
@@ -23,12 +23,13 @@ else:
class AudioTranscriptionRequestData:
"""
Structured data for audio transcription requests.
Attributes:
data: The request data (form data for multipart, json data for regular requests)
files: Optional files dict for multipart form data
content_type: Optional content type override
"""
data: Union[dict, bytes]
files: Optional[dict] = None
content_type: Optional[str] = None
@@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> Union[AudioTranscriptionRequestData, Dict]:
) -> AudioTranscriptionRequestData:
raise NotImplementedError(
"AudioTranscriptionConfig needs a request transformation for audio transcription models"
)
def transform_audio_transcription_response(
self,
raw_response: httpx.Response,
@@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
raise NotImplementedError(
"AudioTranscriptionConfig does not need a response transformation for audio transcription models"
)
def get_provider_specific_params(
self,
@@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
provider_specific_params[key] = value
return provider_specific_params
def _should_exclude_param(
self,
param_name: str,
litellm/llms/bedrock/chat/converse_transformation.py
+11 -7
View File
@@ -14,7 +14,7 @@ from litellm._logging import verbose_logger
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -397,7 +397,11 @@ class AmazonConverseConfig(BaseConfig):
for param, value in non_default_params.items():
if param == "response_format" and isinstance(value, dict):
optional_params = self._translate_response_format_param(
value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled
value=value,
model=model,
optional_params=optional_params,
non_default_params=non_default_params,
is_thinking_enabled=is_thinking_enabled,
)
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["maxTokens"] = value
@@ -446,11 +450,11 @@ class AmazonConverseConfig(BaseConfig):
)
return optional_params
def _translate_response_format_param(
self,
value: dict,
model: str,
self,
value: dict,
model: str,
optional_params: dict,
non_default_params: dict,
is_thinking_enabled: bool,
@@ -504,7 +508,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["json_mode"] = True
if non_default_params.get("stream", False) is True:
optional_params["fake_stream"] = True
return optional_params
def update_optional_params_with_thinking_tokens(
litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py
+1 -1
View File
@@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast
from httpx import Response
from litellm import verbose_logger
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
litellm/llms/custom_httpx/llm_http_handler.py
+36 -17
View File
@@ -118,7 +118,6 @@ class BaseLLMHTTPHandler:
response: Optional[httpx.Response] = None
for i in range(max(max_retry_on_unprocessable_entity_error, 1)):
try:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
@@ -2221,7 +2220,9 @@ class BaseLLMHTTPHandler:
if isinstance(transformed_request, dict) and "method" in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
upload_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2233,8 +2234,8 @@ class BaseLLMHTTPHandler:
# Handle traditional file uploads
# Ensure transformed_request is a string for httpx compatibility
if isinstance(transformed_request, bytes):
transformed_request = transformed_request.decode('utf-8')
transformed_request = transformed_request.decode("utf-8")
# Use the HTTP method specified by the provider config
http_method = provider_config.file_upload_http_method.upper()
if http_method == "PUT":
@@ -2314,7 +2315,7 @@ class BaseLLMHTTPHandler:
)
else:
async_httpx_client = client
#########################################################
# Debug Logging
#########################################################
@@ -2330,7 +2331,9 @@ class BaseLLMHTTPHandler:
if isinstance(transformed_request, dict) and "method" in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
upload_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2342,8 +2345,8 @@ class BaseLLMHTTPHandler:
# Handle traditional file uploads
# Ensure transformed_request is a string for httpx compatibility
if isinstance(transformed_request, bytes):
transformed_request = transformed_request.decode('utf-8')
transformed_request = transformed_request.decode("utf-8")
# Use the HTTP method specified by the provider config
http_method = provider_config.file_upload_http_method.upper()
if http_method == "PUT":
@@ -2468,9 +2471,14 @@ class BaseLLMHTTPHandler:
sync_httpx_client = client
try:
if isinstance(transformed_request, dict) and "method" in transformed_request:
if (
isinstance(transformed_request, dict)
and "method" in transformed_request
):
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
batch_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2500,8 +2508,11 @@ class BaseLLMHTTPHandler:
)
# Store original request for response transformation
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data}
litellm_params_with_request = {
**litellm_params,
"original_batch_request": create_batch_data,
}
return provider_config.transform_create_batch_response(
model=model,
raw_response=batch_response,
@@ -2531,7 +2542,7 @@ class BaseLLMHTTPHandler:
)
else:
async_httpx_client = client
#########################################################
# Debug Logging
#########################################################
@@ -2546,9 +2557,14 @@ class BaseLLMHTTPHandler:
)
try:
if isinstance(transformed_request, dict) and "method" in transformed_request:
if (
isinstance(transformed_request, dict)
and "method" in transformed_request
):
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
batch_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2578,8 +2594,11 @@ class BaseLLMHTTPHandler:
)
# Store original request for response transformation (for async version)
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}}
litellm_params_with_request = {
**litellm_params,
"original_batch_request": create_batch_data or {},
}
return provider_config.transform_create_batch_response(
model=model,
raw_response=batch_response,
litellm/llms/hosted_vllm/transcriptions/transformation.py
+72
View File
@@ -0,0 +1,72 @@
"""
Transformation logic for Hosted VLLM rerank
"""
from typing import Optional, Union
import httpx
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.transcriptions.whisper_transformation import (
OpenAIWhisperAudioTranscriptionConfig,
)
from litellm.types.utils import FileTypes
class HostedVLLMAudioTranscriptionError(BaseLLMException):
def __init__(
self,
status_code: int,
message: str,
headers: Optional[Union[dict, httpx.Headers]] = None,
):
super().__init__(status_code=status_code, message=message, headers=headers)
class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
def __init__(self) -> None:
pass
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if api_base:
# Remove trailing slashes and ensure clean base URL
api_base = api_base.rstrip("/")
if not api_base.endswith("/v1/audio/transcriptions"):
api_base = f"{api_base}/v1/audio/transcriptions"
return api_base
raise ValueError("api_base must be provided for Hosted VLLM rerank")
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
data = {"model": model, "file": audio_file, **optional_params}
if "response_format" not in data or (
data["response_format"] == "text" or data["response_format"] == "json"
):
data["response_format"] = (
"verbose_json" # ensures 'duration' is received - used for cost calculation
)
return AudioTranscriptionRequestData(
data=data,
)
litellm/llms/ollama/chat/transformation.py
+28 -3
View File
@@ -16,9 +16,18 @@ from httpx._models import Headers, Response
from pydantic import BaseModel
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
convert_content_list_to_str,
extract_images_from_message,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction
from litellm.types.llms.ollama import (
OllamaChatCompletionMessage,
OllamaToolCall,
OllamaToolCallFunction,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
@@ -299,7 +308,23 @@ class OllamaChatConfig(BaseConfig):
)
new_tools.append(ollama_tool_call)
cast(dict, m)["tool_calls"] = new_tools
new_messages.append(m)
reasoning_content, parsed_content = _extract_reasoning_content(
cast(dict, m)
)
content_str = convert_content_list_to_str(cast(AllMessageValues, m))
images = extract_images_from_message(cast(AllMessageValues, m))
ollama_message = OllamaChatCompletionMessage(
role=cast(str, m.get("role")),
)
if reasoning_content is not None:
ollama_message["thinking"] = reasoning_content
if content_str is not None:
ollama_message["content"] = content_str
if images is not None:
ollama_message["images"] = images
new_messages.append(ollama_message)
# Load Config
config = self.get_config()
@@ -361,7 +386,7 @@ class OllamaChatConfig(BaseConfig):
del response_json_message["thinking"]
elif response_json_message.get("content") is not None:
# parse reasoning content from content
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
litellm/llms/ollama/completion/transformation.py
+2 -2
View File
@@ -229,7 +229,7 @@ class OllamaConfig(BaseConfig):
model = model.split("/", 1)[1]
api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
api_key = self.get_api_key()
headers = { "Authorization": f"Bearer {api_key}" } if api_key else {}
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
try:
response = litellm.module_level_client.post(
@@ -279,7 +279,7 @@ class OllamaConfig(BaseConfig):
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
litellm/llms/openai/transcriptions/gpt_transformation.py
+9 -2
View File
@@ -1,5 +1,8 @@
from typing import List
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
)
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
from litellm.types.utils import FileTypes
@@ -27,8 +30,12 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> dict:
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
return {"model": model, "file": audio_file, **optional_params}
data = {"model": model, "file": audio_file, **optional_params}
return AudioTranscriptionRequestData(
data=data,
)
litellm/llms/openai/transcriptions/handler.py
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import Optional, Union
from typing import Optional, Union, cast
import httpx
from openai import AsyncOpenAI, OpenAI
@@ -34,6 +34,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
- call openai_aclient.audio.transcriptions.create by default
"""
try:
raw_response = (
await openai_aclient.audio.transcriptions.with_raw_response.create(
**data, timeout=timeout
@@ -93,15 +94,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
Handle audio transcription request
"""
if provider_config is not None:
data = provider_config.transform_audio_transcription_request(
transformed_data = provider_config.transform_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
)
if not isinstance(data, dict):
raise ValueError("OpenAI transformation route requires a dict")
data = cast(dict, transformed_data.data)
else:
data = {"model": model, "file": audio_file, **optional_params}
litellm/llms/openai/transcriptions/whisper_transformation.py
+60 -8
View File
@@ -1,8 +1,9 @@
from typing import List, Optional, Union
from httpx import Headers
from httpx import Headers, Response
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -11,12 +12,40 @@ from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import FileTypes
from litellm.types.utils import FileTypes, TranscriptionResponse
from ..common_utils import OpenAIError
class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
OPTIONAL
Get the complete url for the request
Some providers need `model` in `api_base`
"""
## get the api base, attach the endpoint - v1/audio/transcriptions
# strip trailing slash if present
api_base = api_base.rstrip("/") if api_base else ""
# if endswith "/v1"
if api_base and api_base.endswith("/v1"):
api_base = f"{api_base}/audio/transcriptions"
else:
api_base = f"{api_base}/v1/audio/transcriptions"
return api_base or ""
def get_supported_openai_params(
self, model: str
) -> List[OpenAIAudioTranscriptionOptionalParams]:
@@ -72,21 +101,22 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> dict:
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
data = {"model": model, "file": audio_file, **optional_params}
if "response_format" not in data or (
data["response_format"] == "text" or data["response_format"] == "json"
):
data[
"response_format"
] = "verbose_json" # ensures 'duration' is received - used for cost calculation
data["response_format"] = (
"verbose_json" # ensures 'duration' is received - used for cost calculation
)
return data
return AudioTranscriptionRequestData(
data=data,
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
@@ -96,3 +126,25 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
message=error_message,
headers=headers,
)
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
try:
raw_response_json = raw_response.json()
except Exception as e:
raise ValueError(
f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}"
)
if any(
key in raw_response_json
for key in TranscriptionResponse.model_fields.keys()
):
return TranscriptionResponse(**raw_response_json)
else:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
raw_response_json,
)
litellm/main.py
+9 -8
View File
@@ -5267,7 +5267,10 @@ def transcription(
model_response = litellm.utils.TranscriptionResponse()
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
) # type: ignore
if dynamic_api_key is not None:
@@ -5283,6 +5286,7 @@ def transcription(
custom_llm_provider=custom_llm_provider,
**non_default_params,
)
litellm_params_dict = get_litellm_params(**kwargs)
litellm_logging_obj.update_environment_variables(
@@ -5347,9 +5351,8 @@ def transcription(
max_retries=max_retries,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
elif custom_llm_provider == "openai" or (
custom_llm_provider in litellm.openai_compatible_providers
):
api_base = (
api_base
@@ -5364,6 +5367,7 @@ def transcription(
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") # type: ignore
response = openai_audio_transcriptions.audio_transcriptions(
model=model,
@@ -5380,10 +5384,7 @@ def transcription(
provider_config=provider_config,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider in [
LlmProviders.DEEPGRAM.value,
LlmProviders.ELEVENLABS.value,
]:
elif provider_config is not None:
response = base_llm_http_handler.audio_transcriptions(
model=model,
audio_file=file,
litellm/proxy/_new_secret_config.yaml
+9 -1
View File
@@ -9,4 +9,12 @@ model_list:
model: openai/*
- model_name: xai-grok-3
litellm_params:
model: xai/grok-3
model: xai/grok-3
- model_name: hosted_vllm/whisper-v3
litellm_params:
model: hosted_vllm/whisper-v3
api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5"
api_key: dummy
litellm/proxy/management_endpoints/common_utils.py
+1 -1
View File
@@ -93,7 +93,7 @@ async def _upsert_budget_and_membership(
create_data["tpm_limit"] = tpm_limit
if rpm_limit is not None:
create_data["rpm_limit"] = rpm_limit
new_budget = await tx.litellm_budgettable.create(
data=create_data,
include={"team_membership": True},
litellm/proxy/management_endpoints/key_management_endpoints.py
+12
View File
@@ -925,6 +925,15 @@ async def prepare_key_update_data(
detail="team_id is required for service account keys. Please specify `team_id` in the request body.",
)
non_default_values = {}
# ADD METADATA FIELDS
# Set Management Endpoint Metadata Fields
for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium:
if getattr(data, field, None) is not None:
_set_object_metadata_field(
object_data=data,
field_name=field,
value=getattr(data, field),
)
for k, v in data_json.items():
if (
k in LiteLLM_ManagementEndpoint_MetadataFields
@@ -1137,6 +1146,9 @@ async def update_key_fn(
change_initiated_by=user_api_key_dict,
llm_router=llm_router,
)
# Set Management Endpoint Metadata Fields
non_default_values = await prepare_key_update_data(
data=data, existing_key_row=existing_key_row
)
litellm/router.py
+10 -6
View File
@@ -4414,7 +4414,7 @@ class Router:
return tpm_key
except Exception as e:
verbose_router_logger.exception(
verbose_router_logger.debug(
"litellm.router.Router::deployment_callback_on_success(): Exception occured - {}".format(
str(e)
)
@@ -4562,8 +4562,10 @@ class Router:
parent_otel_span=parent_otel_span,
ttl=RoutingArgs.ttl.value,
)
def _get_metadata_variable_name_from_kwargs(self, kwargs: dict) -> Literal["metadata", "litellm_metadata"]:
def _get_metadata_variable_name_from_kwargs(
self, kwargs: dict
) -> Literal["metadata", "litellm_metadata"]:
"""
Helper to return what the "metadata" field should be called in the request data
@@ -5672,11 +5674,11 @@ class Router:
)
if supported_openai_params is None:
supported_openai_params = []
# Get mode from database model_info if available, otherwise default to "chat"
db_model_info = model.get("model_info", {})
mode = db_model_info.get("mode", "chat")
model_info = ModelMapInfo(
key=model_group,
max_tokens=None,
@@ -6802,7 +6804,9 @@ class Router:
model=model,
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs),
metadata_variable_name=self._get_metadata_variable_name_from_kwargs(
request_kwargs
),
)
if len(healthy_deployments) == 0:
litellm/types/llms/ollama.py
+9
View File
@@ -27,3 +27,12 @@ class OllamaToolCall(TypedDict):
class OllamaVisionModelObject(TypedDict):
prompt: str
images: List[str]
class OllamaChatCompletionMessage(TypedDict, total=False):
role: Required[str]
content: str
thinking: str
images: List[str]
tool_calls: List[OllamaToolCall]
tool_name: str
litellm/utils.py
+13 -1
View File
@@ -107,7 +107,6 @@ from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_s
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
LiteLLMResponseObjectHandler,
_handle_invalid_parallel_tool_calls,
_parse_content_for_reasoning,
convert_to_model_response_object,
convert_to_streaming_response,
convert_to_streaming_response_async,
@@ -122,6 +121,9 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
ResponseMetadata,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
from litellm.litellm_core_utils.redact_messages import (
LiteLLMLoggingObject,
redact_message_input_output_from_logging,
@@ -2426,6 +2428,8 @@ def get_optional_params_transcription(
# retrieve all parameters passed to the function
passed_params = locals()
passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS")
custom_llm_provider = passed_params.pop("custom_llm_provider")
drop_params = passed_params.pop("drop_params")
special_params = passed_params.pop("kwargs")
@@ -2490,6 +2494,7 @@ def get_optional_params_transcription(
model=model,
drop_params=drop_params if drop_params is not None else False,
)
optional_params = add_provider_specific_params_to_optional_params(
optional_params=optional_params,
passed_params=passed_params,
@@ -4087,6 +4092,7 @@ def add_provider_specific_params_to_optional_params(
"""
Add provider specific params to optional_params
"""
if (
custom_llm_provider
in ["openai", "azure", "text-completion-openai"]
@@ -7213,6 +7219,12 @@ class ProviderConfigManager:
return litellm.OpenAIGPTAudioTranscriptionConfig()
else:
return litellm.OpenAIWhisperAudioTranscriptionConfig()
elif litellm.LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.hosted_vllm.transcriptions.transformation import (
HostedVLLMAudioTranscriptionConfig,
)
return HostedVLLMAudioTranscriptionConfig()
return None
@staticmethod
tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py
+227
View File
@@ -1,6 +1,7 @@
import inspect
import os
import sys
from typing import cast
import pytest
from pydantic import BaseModel
@@ -10,6 +11,7 @@ sys.path.insert(
)
from litellm.llms.ollama.chat.transformation import OllamaChatConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import get_optional_params
@@ -101,3 +103,228 @@ class TestOllamaChatConfigResponseFormat:
# Clean up class attributes
delattr(litellm.OllamaChatConfig, "num_ctx")
delattr(litellm.OllamaChatConfig, "temperature")
def test_transform_request_content_list_to_string(self):
"""Test that content list is properly converted to string in transform_request"""
config = OllamaChatConfig()
# Test message with content as list containing text
messages = cast(
list[AllMessageValues],
[
{
"role": "user",
"content": [
{"type": "text", "text": "Hello "},
{"type": "text", "text": "world!"},
],
}
],
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify content was converted to string
assert len(result["messages"]) == 1
assert result["messages"][0]["content"] == "Hello world!"
assert result["messages"][0]["role"] == "user"
def test_transform_request_content_string_passthrough(self):
"""Test that string content passes through unchanged in transform_request"""
config = OllamaChatConfig()
# Test message with content as string
messages = cast(
list[AllMessageValues], [{"role": "user", "content": "Hello world!"}]
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify string content passes through
assert len(result["messages"]) == 1
assert result["messages"][0]["content"] == "Hello world!"
assert result["messages"][0]["role"] == "user"
def test_transform_request_empty_content_list(self):
"""Test handling of empty content list in transform_request"""
config = OllamaChatConfig()
# Test message with empty content list
messages = cast(list[AllMessageValues], [{"role": "user", "content": []}])
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify empty content becomes empty string
assert len(result["messages"]) == 1
assert result["messages"][0]["content"] == ""
assert result["messages"][0]["role"] == "user"
def test_transform_request_image_extraction(self):
"""Test that images are properly extracted from messages in transform_request"""
config = OllamaChatConfig()
# Test message with images in content list
messages = cast(
list[AllMessageValues],
[
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{
"type": "image_url",
"image_url": {
"url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."
},
},
],
}
],
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify text content was extracted
assert len(result["messages"]) == 1
assert result["messages"][0]["content"] == "What's in this image?"
assert result["messages"][0]["role"] == "user"
# Verify image was extracted to images list
assert "images" in result["messages"][0]
assert len(result["messages"][0]["images"]) == 1
assert (
result["messages"][0]["images"][0]
== "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."
)
def test_transform_request_multiple_images_extraction(self):
"""Test extraction of multiple images from a single message"""
config = OllamaChatConfig()
# Test message with multiple images
messages = cast(
list[AllMessageValues],
[
{
"role": "user",
"content": [
{"type": "text", "text": "Compare these images:"},
{
"type": "image_url",
"image_url": {
"url": "data:image/jpeg;base64,image1data..."
},
},
{"type": "text", "text": " and "},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,image2data..."},
},
],
}
],
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify text content was combined
assert result["messages"][0]["content"] == "Compare these images: and "
# Verify both images were extracted
assert "images" in result["messages"][0]
assert len(result["messages"][0]["images"]) == 2
assert (
result["messages"][0]["images"][0] == "data:image/jpeg;base64,image1data..."
)
assert (
result["messages"][0]["images"][1] == "data:image/png;base64,image2data..."
)
def test_transform_request_image_url_as_string(self):
"""Test handling of image_url as direct string (edge case)"""
config = OllamaChatConfig()
# Test message with image_url as string (edge case from extract_images_from_message)
messages = cast(
list[AllMessageValues],
[
{
"role": "user",
"content": [
{"type": "text", "text": "Check this:"},
{
"type": "image_url",
"image_url": "https://example.com/image.jpg",
},
],
}
],
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify image URL was extracted
assert "images" in result["messages"][0]
assert len(result["messages"][0]["images"]) == 1
assert result["messages"][0]["images"][0] == "https://example.com/image.jpg"
def test_transform_request_no_images_no_images_key(self):
"""Test that messages without images don't have images key"""
config = OllamaChatConfig()
# Test message with no images
messages = cast(
list[AllMessageValues],
[{"role": "user", "content": [{"type": "text", "text": "Just text here"}]}],
)
result = config.transform_request(
model="llama2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# Verify no images key when no images present
assert result["messages"][0]["content"] == "Just text here"
# Since extract_images_from_message returns empty list [] when no images found,
# and the code checks "if images is not None", an empty list will still be set
assert "images" in result["messages"][0]
assert result["messages"][0]["images"] == []