diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d1166e157b..4f2f43f0de 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -58,6 +58,7 @@ def get_litellm_params( async_call: Optional[bool] = None, ssl_verify: Optional[bool] = None, merge_reasoning_content_in_choices: Optional[bool] = None, + api_version: Optional[str] = None, max_retries: Optional[int] = None, **kwargs, ) -> dict: @@ -100,6 +101,7 @@ def get_litellm_params( "async_call": async_call, "ssl_verify": ssl_verify, "merge_reasoning_content_in_choices": merge_reasoning_content_in_choices, + "api_version": api_version, "azure_ad_token": kwargs.get("azure_ad_token"), "tenant_id": kwargs.get("tenant_id"), "client_id": kwargs.get("client_id"), diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 625704dbea..212db1853b 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -29,6 +29,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 2ef5285ac6..154f345537 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -67,6 +67,7 @@ class AzureAIStudioConfig(OpenAIConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ @@ -92,12 +93,14 @@ class AzureAIStudioConfig(OpenAIConfig): original_url = httpx.URL(api_base) # Extract api_version or use default - api_version = cast(Optional[str], optional_params.get("api_version")) + api_version = cast(Optional[str], litellm_params.get("api_version")) - # Check if 'api-version' is already present - if "api-version" not in original_url.params and api_version: - # Add api_version to optional_params - original_url.params["api-version"] = api_version + # Create a new dictionary with existing params + query_params = dict(original_url.params) + + # Add api_version if needed + if "api-version" not in query_params and api_version: + query_params["api-version"] = api_version # Add the path to the base URL if "services.ai.azure.com" in api_base: @@ -109,8 +112,7 @@ class AzureAIStudioConfig(OpenAIConfig): api_base=api_base, ending_path="/chat/completions" ) - # Convert optional_params to query parameters - query_params = original_url.params + # Use the new query_params dictionary final_url = httpx.URL(new_url).copy_with(params=query_params) return str(final_url) diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 66140455d9..e550c574e2 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -30,6 +30,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 8327a10464..1b5a6bc58e 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -270,6 +270,7 @@ class BaseConfig(ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index ca258c2562..9432f02da1 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -31,6 +31,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index 940c6bf225..68c0a7c05a 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -45,6 +45,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index dcb53bea94..4d1cd6eebb 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -36,6 +36,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 5414429d4c..133eb659df 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -76,6 +76,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 555e3c21f4..83c7483df9 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -79,6 +79,7 @@ class CloudflareChatConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 4a9e07016f..c865fee17e 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -234,6 +234,7 @@ class BaseLLMAIOHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=stream, ) @@ -483,6 +484,7 @@ class BaseLLMAIOHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=False, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6f8fd347c1..01fe36acda 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -242,6 +242,7 @@ class BaseLLMHTTPHandler: model=model, optional_params=optional_params, stream=stream, + litellm_params=litellm_params, ) data = provider_config.transform_request( @@ -612,6 +613,7 @@ class BaseLLMHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, ) data = provider_config.transform_embedding_request( @@ -907,6 +909,7 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, atranscription: bool = False, headers: dict = {}, + litellm_params: dict = {}, ) -> TranscriptionResponse: provider_config = ProviderConfigManager.get_provider_audio_transcription_config( model=model, provider=litellm.LlmProviders(custom_llm_provider) @@ -930,6 +933,7 @@ class BaseLLMHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, ) # Handle the audio file based on type diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index c8dbd688cc..06296736ea 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -103,6 +103,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 747129ddd8..180cf7dc69 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -40,6 +40,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 283b2a2437..4a7a3556ae 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -356,6 +356,7 @@ class OllamaConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 1f34d63681..8974a2a074 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -291,6 +291,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index ac886e915c..821fc9b7f1 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -230,7 +230,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): logging_obj, optional_params: dict, acompletion=None, - litellm_params=None, + litellm_params: dict = {}, logger_fn=None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -337,7 +337,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): timeout=timeout, base_model=base_model, client=client, - json_mode=json_mode + json_mode=json_mode, ) else: ## COMPLETION CALL diff --git a/litellm/llms/replicate/chat/handler.py b/litellm/llms/replicate/chat/handler.py index e7d0d383e2..f52eb2ee05 100644 --- a/litellm/llms/replicate/chat/handler.py +++ b/litellm/llms/replicate/chat/handler.py @@ -169,7 +169,10 @@ def completion( ) # for pricing this must remain right before calling api prediction_url = replicate_config.get_complete_url( - api_base=api_base, model=model, optional_params=optional_params + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, ) ## COMPLETION CALL @@ -243,7 +246,10 @@ async def async_completion( ) -> Union[ModelResponse, CustomStreamWrapper]: prediction_url = replicate_config.get_complete_url( - api_base=api_base, model=model, optional_params=optional_params + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, ) async_handler = get_async_httpx_client( llm_provider=litellm.LlmProviders.REPLICATE, diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 39aaad6808..75cfe6ced7 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -141,6 +141,7 @@ class ReplicateConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: version_id = self.model_to_version_id(model) diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index 112c3a8f64..8b95deed04 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -55,6 +55,7 @@ class TopazImageVariationConfig(BaseImageVariationConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: api_base = api_base or "https://api.topazlabs.com" diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 4037c32365..56151f89ef 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -72,6 +72,7 @@ class TritonConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/voyage/embedding/transformation.py b/litellm/llms/voyage/embedding/transformation.py index 623dfe73af..51abc9e43a 100644 --- a/litellm/llms/voyage/embedding/transformation.py +++ b/litellm/llms/voyage/embedding/transformation.py @@ -43,6 +43,7 @@ class VoyageEmbeddingConfig(BaseEmbeddingConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base: diff --git a/litellm/llms/watsonx/chat/handler.py b/litellm/llms/watsonx/chat/handler.py index fd195214db..8ea19d413e 100644 --- a/litellm/llms/watsonx/chat/handler.py +++ b/litellm/llms/watsonx/chat/handler.py @@ -31,7 +31,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): logging_obj, optional_params: dict, acompletion=None, - litellm_params=None, + litellm_params: dict = {}, headers: Optional[dict] = None, logger_fn=None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -63,6 +63,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=optional_params.get("stream", False), ) diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index d5e0ed6544..f253da6f5b 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -83,6 +83,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 7a4df23944..f414354e2a 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -318,6 +318,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py index 69c1f8fffa..359137ee5e 100644 --- a/litellm/llms/watsonx/embed/transformation.py +++ b/litellm/llms/watsonx/embed/transformation.py @@ -54,6 +54,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) diff --git a/litellm/main.py b/litellm/main.py index 00174ca435..84ad92dfe0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1162,6 +1162,7 @@ def completion( # type: ignore # noqa: PLR0915 merge_reasoning_content_in_choices=kwargs.get( "merge_reasoning_content_in_choices", None ), + api_version=api_version, azure_ad_token=kwargs.get("azure_ad_token"), tenant_id=kwargs.get("tenant_id"), client_id=kwargs.get("client_id"), diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3c6942a582..70f8623597 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1021,6 +1021,120 @@ "input_cost_per_character": 0.000030, "litellm_provider": "openai" }, + "azure/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000006, + "input_cost_per_audio_token": 0.00001, + "cache_read_input_token_cost": 0.0000003, + "cache_creation_input_audio_token_cost": 0.0000003, + "output_cost_per_token": 0.0000024, + "output_cost_per_audio_token": 0.00002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "input_cost_per_audio_token": 0.0001, + "cache_read_input_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.00002, + "output_cost_per_token": 0.00002, + "output_cost_per_audio_token": 0.0002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.00011, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.00022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.00011, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.00022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/o3-mini-2025-01-31": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -1034,6 +1148,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/tts-1": { "mode": "audio_speech", "input_cost_per_character": 0.000015, @@ -1092,6 +1236,38 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/o1": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -1122,6 +1298,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/o1-preview": { "max_tokens": 32768, "max_input_tokens": 128000, @@ -1150,6 +1356,34 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/gpt-4o": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1195,6 +1429,36 @@ "supports_vision": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1225,6 +1489,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/global-standard/gpt-4o-2024-11-20": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -1285,6 +1581,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index ecf2346a3a..9436e8af0f 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -14,5 +14,4 @@ credential_list: description: "Azure API Key and Base URL" type: "azure" required: true - default: "azure" - + default: "azure" \ No newline at end of file diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d70e4be21f..4824ceec46 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1021,6 +1021,120 @@ "input_cost_per_character": 0.000030, "litellm_provider": "openai" }, + "azure/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000006, + "input_cost_per_audio_token": 0.00001, + "cache_read_input_token_cost": 0.0000003, + "cache_creation_input_audio_token_cost": 0.0000003, + "output_cost_per_token": 0.0000024, + "output_cost_per_audio_token": 0.00002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "input_cost_per_audio_token": 0.0001, + "cache_read_input_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.00002, + "output_cost_per_token": 0.00002, + "output_cost_per_audio_token": 0.0002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.00011, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.00022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.00011, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.00022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/o3-mini-2025-01-31": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -1034,6 +1148,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/tts-1": { "mode": "audio_speech", "input_cost_per_character": 0.000015, @@ -1092,6 +1236,38 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/o1": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -1122,6 +1298,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/o1-preview": { "max_tokens": 32768, "max_input_tokens": 128000, @@ -1150,6 +1356,34 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/gpt-4o": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1195,6 +1429,36 @@ "supports_vision": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1225,6 +1489,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/global-standard/gpt-4o-2024-11-20": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -1285,6 +1581,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/tests/litellm_utils_tests/test_supports_tool_choice.py b/tests/litellm_utils_tests/test_supports_tool_choice.py index 3b512f41f9..cfa190f74b 100644 --- a/tests/litellm_utils_tests/test_supports_tool_choice.py +++ b/tests/litellm_utils_tests/test_supports_tool_choice.py @@ -117,7 +117,9 @@ async def test_supports_tool_choice(): """ # Load model prices litellm._turn_on_debug() - with open("./model_prices_and_context_window.json", "r") as f: + local_path = "../../model_prices_and_context_window.json" + prod_path = "./model_prices_and_context_window.json" + with open(prod_path, "r") as f: model_prices = json.load(f) litellm.model_cost = model_prices config_manager = ProviderConfigManager() @@ -133,6 +135,8 @@ async def test_supports_tool_choice(): or any(provider in model_name for provider in OLD_PROVIDERS) or model_info["litellm_provider"] in OLD_PROVIDERS or model_name in block_list + or "azure/eu" in model_name + or "azure/us" in model_name ): continue diff --git a/tests/llm_translation/test_azure_ai.py b/tests/llm_translation/test_azure_ai.py index c22c9edafa..6d4284cd86 100644 --- a/tests/llm_translation/test_azure_ai.py +++ b/tests/llm_translation/test_azure_ai.py @@ -159,6 +159,32 @@ def test_azure_ai_services_handler(api_base, expected_url): assert mock_client.call_args.kwargs["url"] == expected_url +def test_azure_ai_services_with_api_version(): + from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post") as mock_client: + try: + response = litellm.completion( + model="azure_ai/Meta-Llama-3.1-70B-Instruct", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="my-fake-api-key", + api_version="2024-05-01-preview", + api_base="https://litellm8397336933.services.ai.azure.com/models", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + assert mock_client.call_args.kwargs["headers"]["api-key"] == "my-fake-api-key" + assert ( + mock_client.call_args.kwargs["url"] + == "https://litellm8397336933.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview" + ) + + def test_completion_azure_ai_command_r(): try: import os diff --git a/tests/llm_translation/test_unit_test_bedrock_invoke.py b/tests/llm_translation/test_unit_test_bedrock_invoke.py index da9ad71264..4138073183 100644 --- a/tests/llm_translation/test_unit_test_bedrock_invoke.py +++ b/tests/llm_translation/test_unit_test_bedrock_invoke.py @@ -28,6 +28,7 @@ def test_get_complete_url_basic(bedrock_transformer): model="anthropic.claude-v2", optional_params={}, stream=False, + litellm_params={}, ) assert ( @@ -43,6 +44,7 @@ def test_get_complete_url_streaming(bedrock_transformer): model="anthropic.claude-v2", optional_params={}, stream=True, + litellm_params={}, ) assert (