tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-25 00:26:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(hosted_vllm/): transcription endpoint support

Browse Source 
Closes https://github.com/BerriAI/litellm/issues/361#issuecomment-3244548055
This commit is contained in:
Krrish Dholakia
2025-09-12 17:15:14 -07:00
parent 1bbbacea00
commit 82091de393
10 changed files with 229 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/constants.py
+7 -3
View File
@@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
)
DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4)
os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
)
DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
SQS_SEND_MESSAGE_ACTION = "SendMessage"
@@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128)
)
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512)
os.getenv(
"DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512
)
)
# Generic fallback for unknown models
@@ -949,7 +951,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000))
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
)
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
litellm/llms/base_llm/audio_transcription/transformation.py
+4 -6
View File
@@ -23,12 +23,13 @@ else:
class AudioTranscriptionRequestData:
"""
Structured data for audio transcription requests.
Attributes:
data: The request data (form data for multipart, json data for regular requests)
files: Optional files dict for multipart form data
content_type: Optional content type override
"""
data: Union[dict, bytes]
files: Optional[dict] = None
content_type: Optional[str] = None
@@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> Union[AudioTranscriptionRequestData, Dict]:
) -> AudioTranscriptionRequestData:
raise NotImplementedError(
"AudioTranscriptionConfig needs a request transformation for audio transcription models"
)
def transform_audio_transcription_response(
self,
raw_response: httpx.Response,
@@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
raise NotImplementedError(
"AudioTranscriptionConfig does not need a response transformation for audio transcription models"
)
def get_provider_specific_params(
self,
@@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
provider_specific_params[key] = value
return provider_specific_params
def _should_exclude_param(
self,
param_name: str,
litellm/llms/custom_httpx/llm_http_handler.py
+36 -16
View File
@@ -2221,7 +2221,9 @@ class BaseLLMHTTPHandler:
if isinstance(transformed_request, dict) and "method" in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
upload_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2233,8 +2235,8 @@ class BaseLLMHTTPHandler:
# Handle traditional file uploads
# Ensure transformed_request is a string for httpx compatibility
if isinstance(transformed_request, bytes):
transformed_request = transformed_request.decode('utf-8')
transformed_request = transformed_request.decode("utf-8")
# Use the HTTP method specified by the provider config
http_method = provider_config.file_upload_http_method.upper()
if http_method == "PUT":
@@ -2310,7 +2312,7 @@ class BaseLLMHTTPHandler:
)
else:
async_httpx_client = client
#########################################################
# Debug Logging
#########################################################
@@ -2326,7 +2328,9 @@ class BaseLLMHTTPHandler:
if isinstance(transformed_request, dict) and "method" in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
upload_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2338,8 +2342,8 @@ class BaseLLMHTTPHandler:
# Handle traditional file uploads
# Ensure transformed_request is a string for httpx compatibility
if isinstance(transformed_request, bytes):
transformed_request = transformed_request.decode('utf-8')
transformed_request = transformed_request.decode("utf-8")
# Use the HTTP method specified by the provider config
http_method = provider_config.file_upload_http_method.upper()
if http_method == "PUT":
@@ -2460,9 +2464,14 @@ class BaseLLMHTTPHandler:
sync_httpx_client = client
try:
if isinstance(transformed_request, dict) and "method" in transformed_request:
if (
isinstance(transformed_request, dict)
and "method" in transformed_request
):
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
batch_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2492,8 +2501,11 @@ class BaseLLMHTTPHandler:
)
# Store original request for response transformation
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data}
litellm_params_with_request = {
**litellm_params,
"original_batch_request": create_batch_data,
}
return provider_config.transform_create_batch_response(
model=None,
raw_response=batch_response,
@@ -2522,7 +2534,7 @@ class BaseLLMHTTPHandler:
)
else:
async_httpx_client = client
#########################################################
# Debug Logging
#########################################################
@@ -2537,9 +2549,14 @@ class BaseLLMHTTPHandler:
)
try:
if isinstance(transformed_request, dict) and "method" in transformed_request:
if (
isinstance(transformed_request, dict)
and "method" in transformed_request
):
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
batch_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
@@ -2569,8 +2586,11 @@ class BaseLLMHTTPHandler:
)
# Store original request for response transformation (for async version)
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}}
litellm_params_with_request = {
**litellm_params,
"original_batch_request": create_batch_data or {},
}
return provider_config.transform_create_batch_response(
model=None,
raw_response=batch_response,
litellm/llms/hosted_vllm/transcriptions/transformation.py
+86
View File
@@ -0,0 +1,86 @@
"""
Transformation logic for Hosted VLLM rerank
"""
import uuid
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.llms.openai.transcriptions.whisper_transformation import (
OpenAIWhisperAudioTranscriptionConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.rerank import (
OptionalRerankParams,
RerankBilledUnits,
RerankRequest,
RerankResponse,
RerankResponseDocument,
RerankResponseMeta,
RerankResponseResult,
RerankTokens,
)
from litellm.types.utils import FileTypes
class HostedVLLMAudioTranscriptionError(BaseLLMException):
def __init__(
self,
status_code: int,
message: str,
headers: Optional[Union[dict, httpx.Headers]] = None,
):
super().__init__(status_code=status_code, message=message, headers=headers)
class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
def __init__(self) -> None:
pass
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if api_base:
# Remove trailing slashes and ensure clean base URL
api_base = api_base.rstrip("/")
if not api_base.endswith("/v1/audio/transcriptions"):
api_base = f"{api_base}/v1/audio/transcriptions"
return api_base
raise ValueError("api_base must be provided for Hosted VLLM rerank")
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
data = {"model": model, "file": audio_file, **optional_params}
if "response_format" not in data or (
data["response_format"] == "text" or data["response_format"] == "json"
):
data["response_format"] = (
"verbose_json" # ensures 'duration' is received - used for cost calculation
)
return AudioTranscriptionRequestData(
data=data,
)
litellm/llms/openai/transcriptions/gpt_transformation.py
+9 -2
View File
@@ -1,5 +1,8 @@
from typing import List
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
)
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
from litellm.types.utils import FileTypes
@@ -27,8 +30,12 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> dict:
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
return {"model": model, "file": audio_file, **optional_params}
data = {"model": model, "file": audio_file, **optional_params}
return AudioTranscriptionRequestData(
data=data,
)
litellm/llms/openai/transcriptions/handler.py
+3 -4
View File
@@ -1,4 +1,4 @@
from typing import Optional, Union
from typing import Optional, Union, cast
import httpx
from openai import AsyncOpenAI, OpenAI
@@ -93,15 +93,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
Handle audio transcription request
"""
if provider_config is not None:
data = provider_config.transform_audio_transcription_request(
transformed_data = provider_config.transform_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
)
if not isinstance(data, dict):
raise ValueError("OpenAI transformation route requires a dict")
data = cast(dict, transformed_data.data)
else:
data = {"model": model, "file": audio_file, **optional_params}
litellm/llms/openai/transcriptions/whisper_transformation.py
+32 -7
View File
@@ -1,8 +1,9 @@
from typing import List, Optional, Union
from httpx import Headers
from httpx import Headers, Response
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -11,7 +12,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import FileTypes
from litellm.types.utils import FileTypes, TranscriptionResponse
from ..common_utils import OpenAIError
@@ -72,7 +73,7 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> dict:
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
"""
@@ -82,11 +83,13 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
if "response_format" not in data or (
data["response_format"] == "text" or data["response_format"] == "json"
):
data[
"response_format"
] = "verbose_json" # ensures 'duration' is received - used for cost calculation
data["response_format"] = (
"verbose_json" # ensures 'duration' is received - used for cost calculation
)
return data
return AudioTranscriptionRequestData(
data=data,
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
@@ -96,3 +99,25 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
message=error_message,
headers=headers,
)
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
try:
raw_response_json = raw_response.json()
except Exception as e:
raise ValueError(
f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}"
)
if any(
key in raw_response_json
for key in TranscriptionResponse.model_fields.keys()
):
return TranscriptionResponse(**raw_response_json)
else:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
raw_response_json,
)
litellm/main.py
+33 -21
View File
@@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from .llms.custom_llm import CustomLLM, custom_chat_llm_router
from .llms.databricks.embed.handler import DatabricksEmbeddingHandler
from .llms.deprecated_providers import aleph_alpha, palm
from .llms.gemini.common_utils import get_api_key_from_env
from .llms.groq.chat.handler import GroqChatCompletion
from .llms.heroku.chat.transformation import HerokuChatConfig
from .llms.gemini.common_utils import get_api_key_from_env
from .llms.huggingface.embedding.handler import HuggingFaceEmbedding
from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
from .llms.oci.chat.transformation import OCIChatConfig
@@ -358,7 +358,9 @@ async def acompletion(
logprobs: Optional[bool] = None,
top_logprobs: Optional[int] = None,
deployment_id=None,
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
reasoning_effort: Optional[
Literal["none", "minimal", "low", "medium", "high", "default"]
] = None,
safety_identifier: Optional[str] = None,
# set api_base, api_version, api_key
base_url: Optional[str] = None,
@@ -504,7 +506,9 @@ async def acompletion(
}
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None)
model=model,
custom_llm_provider=custom_llm_provider,
api_base=completion_kwargs.get("base_url", None),
)
fallbacks = fallbacks or litellm.model_fallbacks
@@ -899,7 +903,9 @@ def completion( # type: ignore # noqa: PLR0915
logit_bias: Optional[dict] = None,
user: Optional[str] = None,
# openai v1.0+ new params
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
reasoning_effort: Optional[
Literal["none", "minimal", "low", "medium", "high", "default"]
] = None,
response_format: Optional[Union[dict, Type[BaseModel]]] = None,
seed: Optional[int] = None,
tools: Optional[List] = None,
@@ -1116,10 +1122,12 @@ def completion( # type: ignore # noqa: PLR0915
)
if provider_specific_header is not None:
headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
))
headers.update(
ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
)
)
if model_response is not None and hasattr(model_response, "_hidden_params"):
model_response._hidden_params["custom_llm_provider"] = custom_llm_provider
@@ -2712,9 +2720,7 @@ def completion( # type: ignore # noqa: PLR0915
)
api_key = (
api_key
or litellm.api_key
or get_secret("VERCEL_AI_GATEWAY_API_KEY")
api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
)
vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
@@ -2730,7 +2736,7 @@ def completion( # type: ignore # noqa: PLR0915
vercel_headers.update(_headers)
headers = vercel_headers
## Load Config
config = litellm.VercelAIGatewayConfig.get_config()
for k, v in config.items():
@@ -3712,7 +3718,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
func_with_context = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None)
model=model,
custom_llm_provider=custom_llm_provider,
api_base=kwargs.get("api_base", None),
)
# Await normally
@@ -5338,9 +5346,9 @@ def transcription(
max_retries=max_retries,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
elif custom_llm_provider == "openai" or (
custom_llm_provider in litellm.openai_compatible_providers
and provider_config is None
):
api_base = (
api_base
@@ -5371,10 +5379,7 @@ def transcription(
provider_config=provider_config,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider in [
LlmProviders.DEEPGRAM.value,
LlmProviders.ELEVENLABS.value,
]:
elif provider_config is not None:
response = base_llm_http_handler.audio_transcriptions(
model=model,
audio_file=file,
@@ -5780,7 +5785,14 @@ async def ahealth_check(
input=input or ["test"],
),
"audio_speech": lambda: litellm.aspeech(
**{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})},
**{
**_filter_model_params(model_params),
**(
{"voice": "alloy"}
if "voice" not in _filter_model_params(model_params)
else {}
),
},
input=prompt or "test",
),
"audio_transcription": lambda: litellm.atranscription(
litellm/proxy/_new_secret_config.yaml
+5
View File
@@ -7,3 +7,8 @@ model_list:
- model_name: wildcard_models/*
litellm_params:
model: openai/*
- model_name: hosted_vllm/*
litellm_params:
model: hosted_vllm/*
api_base: https://webhook.site/6fbe498e-88b5-4a5f-8f07-edb9806c1937
api_key: fake-key
litellm/utils.py
+14 -4
View File
@@ -2437,7 +2437,7 @@ def get_optional_params_transcription(
"prompt": None,
"response_format": None,
"temperature": None, # openai defaults this to 0
"timestamp_granularities": None
"timestamp_granularities": None,
}
non_default_params = {
@@ -2505,7 +2505,7 @@ def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str:
"""Map OpenAI size parameter to Vertex AI aspectRatio."""
if size is None:
return "1:1"
# Map OpenAI size strings to Vertex AI aspect ratio strings
# Vertex AI accepts: "1:1", "9:16", "16:9", "4:3", "3:4"
size_to_aspect_ratio = {
@@ -2515,7 +2515,9 @@ def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str:
"1792x1024": "16:9", # Landscape
"1024x1792": "9:16", # Portrait
}
return size_to_aspect_ratio.get(size, "1:1") # Default to square if size not recognized
return size_to_aspect_ratio.get(
size, "1:1"
) # Default to square if size not recognized
def get_optional_params_image_gen(
@@ -2631,7 +2633,9 @@ def get_optional_params_image_gen(
# Map OpenAI size parameter to Vertex AI aspectRatio
if size is not None:
optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(size)
optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(
size
)
openai_params: list[str] = list(default_params.keys())
if provider_config is not None:
@@ -7209,6 +7213,12 @@ class ProviderConfigManager:
return litellm.OpenAIGPTAudioTranscriptionConfig()
else:
return litellm.OpenAIWhisperAudioTranscriptionConfig()
elif litellm.LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.hosted_vllm.transcriptions.transformation import (
HostedVLLMAudioTranscriptionConfig,
)
return HostedVLLMAudioTranscriptionConfig()
return None
@staticmethod