From 738c0b873d653ff0694bc27a0bc3e1e66340c1d1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 15:16:42 -0700 Subject: [PATCH 01/10] fix(azure_ai/transformation.py): support passing api version to azure ai services endpoint Fixes https://github.com/BerriAI/litellm/issues/7275 --- .../litellm_core_utils/get_litellm_params.py | 2 ++ litellm/llms/azure_ai/chat/transformation.py | 16 +++++++----- litellm/llms/base_llm/chat/transformation.py | 1 + litellm/llms/custom_httpx/llm_http_handler.py | 1 + litellm/main.py | 1 + tests/llm_translation/test_azure_ai.py | 26 +++++++++++++++++++ 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index cf62375f33..fcf83d17a2 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -58,6 +58,7 @@ def get_litellm_params( async_call: Optional[bool] = None, ssl_verify: Optional[bool] = None, merge_reasoning_content_in_choices: Optional[bool] = None, + api_version: Optional[str] = None, **kwargs, ) -> dict: litellm_params = { @@ -99,5 +100,6 @@ def get_litellm_params( "async_call": async_call, "ssl_verify": ssl_verify, "merge_reasoning_content_in_choices": merge_reasoning_content_in_choices, + "api_version": api_version, } return litellm_params diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 2ef5285ac6..154f345537 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -67,6 +67,7 @@ class AzureAIStudioConfig(OpenAIConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ @@ -92,12 +93,14 @@ class AzureAIStudioConfig(OpenAIConfig): original_url = httpx.URL(api_base) # Extract api_version or use default - api_version = cast(Optional[str], optional_params.get("api_version")) + api_version = cast(Optional[str], litellm_params.get("api_version")) - # Check if 'api-version' is already present - if "api-version" not in original_url.params and api_version: - # Add api_version to optional_params - original_url.params["api-version"] = api_version + # Create a new dictionary with existing params + query_params = dict(original_url.params) + + # Add api_version if needed + if "api-version" not in query_params and api_version: + query_params["api-version"] = api_version # Add the path to the base URL if "services.ai.azure.com" in api_base: @@ -109,8 +112,7 @@ class AzureAIStudioConfig(OpenAIConfig): api_base=api_base, ending_path="/chat/completions" ) - # Convert optional_params to query parameters - query_params = original_url.params + # Use the new query_params dictionary final_url = httpx.URL(new_url).copy_with(params=query_params) return str(final_url) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 8327a10464..1b5a6bc58e 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -270,6 +270,7 @@ class BaseConfig(ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9d67fd1a85..f3600923c6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -234,6 +234,7 @@ class BaseLLMHTTPHandler: model=model, optional_params=optional_params, stream=stream, + litellm_params=litellm_params, ) data = provider_config.transform_request( diff --git a/litellm/main.py b/litellm/main.py index b90030a6bb..a6a1a7c7d1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1162,6 +1162,7 @@ def completion( # type: ignore # noqa: PLR0915 merge_reasoning_content_in_choices=kwargs.get( "merge_reasoning_content_in_choices", None ), + api_version=api_version, ) logging.update_environment_variables( model=model, diff --git a/tests/llm_translation/test_azure_ai.py b/tests/llm_translation/test_azure_ai.py index c22c9edafa..6d4284cd86 100644 --- a/tests/llm_translation/test_azure_ai.py +++ b/tests/llm_translation/test_azure_ai.py @@ -159,6 +159,32 @@ def test_azure_ai_services_handler(api_base, expected_url): assert mock_client.call_args.kwargs["url"] == expected_url +def test_azure_ai_services_with_api_version(): + from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post") as mock_client: + try: + response = litellm.completion( + model="azure_ai/Meta-Llama-3.1-70B-Instruct", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="my-fake-api-key", + api_version="2024-05-01-preview", + api_base="https://litellm8397336933.services.ai.azure.com/models", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + assert mock_client.call_args.kwargs["headers"]["api-key"] == "my-fake-api-key" + assert ( + mock_client.call_args.kwargs["url"] + == "https://litellm8397336933.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview" + ) + + def test_completion_azure_ai_command_r(): try: import os From 88e9edf7db562b85ee0f85168059185809d370ae Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 15:23:38 -0700 Subject: [PATCH 02/10] refactor: update method signature --- litellm/llms/aiohttp_openai/chat/transformation.py | 1 + .../base_llm/audio_transcription/transformation.py | 1 + litellm/llms/base_llm/completion/transformation.py | 1 + litellm/llms/base_llm/embedding/transformation.py | 1 + .../llms/base_llm/image_variations/transformation.py | 1 + .../base_invoke_transformation.py | 1 + litellm/llms/cloudflare/chat/transformation.py | 1 + litellm/llms/custom_httpx/aiohttp_handler.py | 2 ++ litellm/llms/custom_httpx/llm_http_handler.py | 3 +++ .../deepgram/audio_transcription/transformation.py | 1 + litellm/llms/deepseek/chat/transformation.py | 1 + litellm/llms/ollama/completion/transformation.py | 1 + litellm/llms/openai/chat/gpt_transformation.py | 1 + litellm/llms/openai_like/chat/handler.py | 4 ++-- litellm/llms/replicate/chat/handler.py | 10 ++++++++-- litellm/llms/replicate/chat/transformation.py | 1 + litellm/llms/topaz/image_variations/transformation.py | 1 + litellm/llms/triton/completion/transformation.py | 1 + litellm/llms/voyage/embedding/transformation.py | 1 + litellm/llms/watsonx/chat/handler.py | 3 ++- litellm/llms/watsonx/chat/transformation.py | 1 + litellm/llms/watsonx/completion/transformation.py | 1 + litellm/llms/watsonx/embed/transformation.py | 1 + 23 files changed, 35 insertions(+), 5 deletions(-) diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 625704dbea..212db1853b 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -29,6 +29,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 66140455d9..e550c574e2 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -30,6 +30,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index ca258c2562..9432f02da1 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -31,6 +31,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index 940c6bf225..68c0a7c05a 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -45,6 +45,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index dcb53bea94..4d1cd6eebb 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -36,6 +36,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 5414429d4c..133eb659df 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -76,6 +76,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 555e3c21f4..83c7483df9 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -79,6 +79,7 @@ class CloudflareChatConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 4a9e07016f..c865fee17e 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -234,6 +234,7 @@ class BaseLLMAIOHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=stream, ) @@ -483,6 +484,7 @@ class BaseLLMAIOHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=False, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f3600923c6..df8b18b81f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -605,6 +605,7 @@ class BaseLLMHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, ) data = provider_config.transform_embedding_request( @@ -900,6 +901,7 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, atranscription: bool = False, headers: dict = {}, + litellm_params: dict = {}, ) -> TranscriptionResponse: provider_config = ProviderConfigManager.get_provider_audio_transcription_config( model=model, provider=litellm.LlmProviders(custom_llm_provider) @@ -923,6 +925,7 @@ class BaseLLMHTTPHandler: api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, ) # Handle the audio file based on type diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index c8dbd688cc..06296736ea 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -103,6 +103,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 747129ddd8..180cf7dc69 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -40,6 +40,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 283b2a2437..4a7a3556ae 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -356,6 +356,7 @@ class OllamaConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 1f34d63681..8974a2a074 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -291,6 +291,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index ac886e915c..821fc9b7f1 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -230,7 +230,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): logging_obj, optional_params: dict, acompletion=None, - litellm_params=None, + litellm_params: dict = {}, logger_fn=None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -337,7 +337,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): timeout=timeout, base_model=base_model, client=client, - json_mode=json_mode + json_mode=json_mode, ) else: ## COMPLETION CALL diff --git a/litellm/llms/replicate/chat/handler.py b/litellm/llms/replicate/chat/handler.py index e7d0d383e2..f52eb2ee05 100644 --- a/litellm/llms/replicate/chat/handler.py +++ b/litellm/llms/replicate/chat/handler.py @@ -169,7 +169,10 @@ def completion( ) # for pricing this must remain right before calling api prediction_url = replicate_config.get_complete_url( - api_base=api_base, model=model, optional_params=optional_params + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, ) ## COMPLETION CALL @@ -243,7 +246,10 @@ async def async_completion( ) -> Union[ModelResponse, CustomStreamWrapper]: prediction_url = replicate_config.get_complete_url( - api_base=api_base, model=model, optional_params=optional_params + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, ) async_handler = get_async_httpx_client( llm_provider=litellm.LlmProviders.REPLICATE, diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 39aaad6808..75cfe6ced7 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -141,6 +141,7 @@ class ReplicateConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: version_id = self.model_to_version_id(model) diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index 112c3a8f64..8b95deed04 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -55,6 +55,7 @@ class TopazImageVariationConfig(BaseImageVariationConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: api_base = api_base or "https://api.topazlabs.com" diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 4037c32365..56151f89ef 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -72,6 +72,7 @@ class TritonConfig(BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base is None: diff --git a/litellm/llms/voyage/embedding/transformation.py b/litellm/llms/voyage/embedding/transformation.py index 623dfe73af..51abc9e43a 100644 --- a/litellm/llms/voyage/embedding/transformation.py +++ b/litellm/llms/voyage/embedding/transformation.py @@ -43,6 +43,7 @@ class VoyageEmbeddingConfig(BaseEmbeddingConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: if api_base: diff --git a/litellm/llms/watsonx/chat/handler.py b/litellm/llms/watsonx/chat/handler.py index fd195214db..8ea19d413e 100644 --- a/litellm/llms/watsonx/chat/handler.py +++ b/litellm/llms/watsonx/chat/handler.py @@ -31,7 +31,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): logging_obj, optional_params: dict, acompletion=None, - litellm_params=None, + litellm_params: dict = {}, headers: Optional[dict] = None, logger_fn=None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -63,6 +63,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): api_base=api_base, model=model, optional_params=optional_params, + litellm_params=litellm_params, stream=optional_params.get("stream", False), ) diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index d5e0ed6544..f253da6f5b 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -83,6 +83,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 7a4df23944..f414354e2a 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -318,6 +318,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py index 69c1f8fffa..359137ee5e 100644 --- a/litellm/llms/watsonx/embed/transformation.py +++ b/litellm/llms/watsonx/embed/transformation.py @@ -54,6 +54,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): api_base: Optional[str], model: str, optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, ) -> str: url = self._get_base_url(api_base=api_base) From 3714694c60df2867f10fb3bd8b2353ff71464683 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 15:32:35 -0700 Subject: [PATCH 03/10] fix: fix method signature in test --- tests/llm_translation/test_unit_test_bedrock_invoke.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/llm_translation/test_unit_test_bedrock_invoke.py b/tests/llm_translation/test_unit_test_bedrock_invoke.py index da9ad71264..4138073183 100644 --- a/tests/llm_translation/test_unit_test_bedrock_invoke.py +++ b/tests/llm_translation/test_unit_test_bedrock_invoke.py @@ -28,6 +28,7 @@ def test_get_complete_url_basic(bedrock_transformer): model="anthropic.claude-v2", optional_params={}, stream=False, + litellm_params={}, ) assert ( @@ -43,6 +44,7 @@ def test_get_complete_url_streaming(bedrock_transformer): model="anthropic.claude-v2", optional_params={}, stream=True, + litellm_params={}, ) assert ( From 1606e72440bb3a0f22195984df3a91e0bc5d4f69 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:04:04 -0700 Subject: [PATCH 04/10] build(model_prices_and_context_window.json): add azure eu/us data zone pricing --- ...odel_prices_and_context_window_backup.json | 30 +++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 14 ++++++++- model_prices_and_context_window.json | 30 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3c6942a582..6b4c2d2690 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1034,6 +1034,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/tts-1": { "mode": "audio_speech", "input_cost_per_character": 0.000015, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 1aed68b09f..96e6c64376 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -2,4 +2,16 @@ model_list: - model_name: amazon.nova-canvas-v1:0 litellm_params: model: bedrock/amazon.nova-canvas-v1:0 - aws_region_name: "us-east-1" \ No newline at end of file + aws_region_name: "us-east-1" + - model_name: gpt-4o-mini-3 + litellm_params: + model: azure/gpt-4o-mini-3 + api_key: os.environ/AZURE_API_KEY + api_base: os.environ/AZURE_API_BASE + model_info: + base_model: azure/eu.gpt-4o-mini-2 + - model_name: gpt-4o-mini-2 + litellm_params: + model: azure/gpt-4o-mini-2 + api_key: os.environ/AZURE_API_KEY + api_base: os.environ/AZURE_API_BASE \ No newline at end of file diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3c6942a582..6b4c2d2690 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1034,6 +1034,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o3-mini-2025-01-31": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_vision": false, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/tts-1": { "mode": "audio_speech", "input_cost_per_character": 0.000015, From a14ac0505c68d49952c9674666a415c1bc4dc02b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:09:49 -0700 Subject: [PATCH 05/10] build(model_prices_and_context_window.json): add azure o1 eu/us pricing --- ...odel_prices_and_context_window_backup.json | 32 +++++++++++++++++++ model_prices_and_context_window.json | 32 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6b4c2d2690..3f9b0b2938 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1122,6 +1122,38 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/o1": { "max_tokens": 100000, "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6b4c2d2690..3f9b0b2938 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1122,6 +1122,38 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-mini-2024-09-12": { + "max_tokens": 65536, + "max_input_tokens": 128000, + "max_output_tokens": 65536, + "input_cost_per_token": 0.00000121, + "input_cost_per_token_batches": 0.000000605, + "output_cost_per_token": 0.00000484, + "output_cost_per_token_batches": 0.00000242, + "cache_read_input_token_cost": 0.000000605, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/o1": { "max_tokens": 100000, "max_input_tokens": 200000, From c5256d3da684c8cde6747a26cfc9b700bef7cff8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:14:12 -0700 Subject: [PATCH 06/10] build(model_prices_and_context_window.json): add o1 data zone pricing --- ...odel_prices_and_context_window_backup.json | 58 +++++++++++++++++++ model_prices_and_context_window.json | 58 +++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3f9b0b2938..f679066a86 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1184,6 +1184,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/o1-preview": { "max_tokens": 32768, "max_input_tokens": 128000, @@ -1212,6 +1242,34 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/gpt-4o": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3f9b0b2938..f679066a86 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1184,6 +1184,36 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/o1-2024-12-17": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/o1-preview": { "max_tokens": 32768, "max_input_tokens": 128000, @@ -1212,6 +1242,34 @@ "supports_vision": false, "supports_prompt_caching": true }, + "azure/us/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "azure/eu/o1-preview-2024-09-12": { + "max_tokens": 32768, + "max_input_tokens": 128000, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, "azure/gpt-4o": { "max_tokens": 4096, "max_input_tokens": 128000, From 84b4d1509a2bd549338095a2f27d3636f266b0ae Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:27:27 -0700 Subject: [PATCH 07/10] build(model_prices_and_context_window.json): add gpt-4o-mini-realtime-preview azure model cost --- ...odel_prices_and_context_window_backup.json | 114 ++++++++++++++++++ model_prices_and_context_window.json | 114 ++++++++++++++++++ 2 files changed, 228 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f679066a86..bbcad25f8a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1021,6 +1021,120 @@ "input_cost_per_character": 0.000030, "litellm_provider": "openai" }, + "azure/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000006, + "input_cost_per_audio_token": 0.00001, + "cache_read_input_token_cost": 0.0000003, + "cache_creation_input_audio_token_cost": 0.0000003, + "output_cost_per_token": 0.0000024, + "output_cost_per_audio_token": 0.00002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "input_cost_per_audio_token": 0.00004, + "cache_read_input_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.0000025, + "output_cost_per_token": 0.00002, + "output_cost_per_audio_token": 0.00008, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.000044, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.00000275, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.000088, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.000044, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.00000275, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.000088, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/o3-mini-2025-01-31": { "max_tokens": 100000, "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f679066a86..bbcad25f8a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1021,6 +1021,120 @@ "input_cost_per_character": 0.000030, "litellm_provider": "openai" }, + "azure/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000006, + "input_cost_per_audio_token": 0.00001, + "cache_read_input_token_cost": 0.0000003, + "cache_creation_input_audio_token_cost": 0.0000003, + "output_cost_per_token": 0.0000024, + "output_cost_per_audio_token": 0.00002, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-mini-realtime-preview-2024-12-17": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000066, + "input_cost_per_audio_token": 0.000011, + "cache_read_input_token_cost": 0.00000033, + "cache_creation_input_audio_token_cost": 0.00000033, + "output_cost_per_token": 0.00000264, + "output_cost_per_audio_token": 0.000022, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "input_cost_per_audio_token": 0.00004, + "cache_read_input_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.0000025, + "output_cost_per_token": 0.00002, + "output_cost_per_audio_token": 0.00008, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/us/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.000044, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.00000275, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.000088, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-realtime-preview-2024-10-01": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000055, + "input_cost_per_audio_token": 0.000044, + "cache_read_input_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.00000275, + "output_cost_per_token": 0.000022, + "output_cost_per_audio_token": 0.000088, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_audio_input": true, + "supports_audio_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/o3-mini-2025-01-31": { "max_tokens": 100000, "max_input_tokens": 200000, From 1250ae6645948f3c0ee493e68265e0f516b09c70 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:40:11 -0700 Subject: [PATCH 08/10] build(model_prices_and_context_window.json): add data zone pricing --- ...odel_prices_and_context_window_backup.json | 112 ++++++++++++++++-- model_prices_and_context_window.json | 112 ++++++++++++++++-- 2 files changed, 206 insertions(+), 18 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bbcad25f8a..70f8623597 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1083,11 +1083,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.000005, - "input_cost_per_audio_token": 0.00004, + "input_cost_per_audio_token": 0.0001, "cache_read_input_token_cost": 0.0000025, - "cache_creation_input_audio_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.00002, "output_cost_per_token": 0.00002, - "output_cost_per_audio_token": 0.00008, + "output_cost_per_audio_token": 0.0002, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1102,11 +1102,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000055, - "input_cost_per_audio_token": 0.000044, + "input_cost_per_audio_token": 0.00011, "cache_read_input_token_cost": 0.00000275, - "cache_creation_input_audio_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, "output_cost_per_token": 0.000022, - "output_cost_per_audio_token": 0.000088, + "output_cost_per_audio_token": 0.00022, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1121,11 +1121,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000055, - "input_cost_per_audio_token": 0.000044, + "input_cost_per_audio_token": 0.00011, "cache_read_input_token_cost": 0.00000275, - "cache_creation_input_audio_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, "output_cost_per_token": 0.000022, - "output_cost_per_audio_token": 0.000088, + "output_cost_per_audio_token": 0.00022, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1429,6 +1429,36 @@ "supports_vision": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1459,6 +1489,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/global-standard/gpt-4o-2024-11-20": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -1519,6 +1581,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bbcad25f8a..70f8623597 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1083,11 +1083,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.000005, - "input_cost_per_audio_token": 0.00004, + "input_cost_per_audio_token": 0.0001, "cache_read_input_token_cost": 0.0000025, - "cache_creation_input_audio_token_cost": 0.0000025, + "cache_creation_input_audio_token_cost": 0.00002, "output_cost_per_token": 0.00002, - "output_cost_per_audio_token": 0.00008, + "output_cost_per_audio_token": 0.0002, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1102,11 +1102,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000055, - "input_cost_per_audio_token": 0.000044, + "input_cost_per_audio_token": 0.00011, "cache_read_input_token_cost": 0.00000275, - "cache_creation_input_audio_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, "output_cost_per_token": 0.000022, - "output_cost_per_audio_token": 0.000088, + "output_cost_per_audio_token": 0.00022, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1121,11 +1121,11 @@ "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000055, - "input_cost_per_audio_token": 0.000044, + "input_cost_per_audio_token": 0.00011, "cache_read_input_token_cost": 0.00000275, - "cache_creation_input_audio_token_cost": 0.00000275, + "cache_creation_input_audio_token_cost": 0.000022, "output_cost_per_token": 0.000022, - "output_cost_per_audio_token": 0.000088, + "output_cost_per_audio_token": 0.00022, "litellm_provider": "azure", "mode": "chat", "supports_function_calling": true, @@ -1429,6 +1429,36 @@ "supports_vision": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "cache_creation_input_token_cost": 0.00000138, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -1459,6 +1489,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-2024-08-06": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/global-standard/gpt-4o-2024-11-20": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -1519,6 +1581,38 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "azure/us/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, + "azure/eu/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000083, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_tool_choice": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, From 81d8a353482cab40d73a1793711d2f9372dd34e2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:43:38 -0700 Subject: [PATCH 09/10] fix: skip datazone models --- tests/litellm_utils_tests/test_supports_tool_choice.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/litellm_utils_tests/test_supports_tool_choice.py b/tests/litellm_utils_tests/test_supports_tool_choice.py index 3b512f41f9..a68aea4f5a 100644 --- a/tests/litellm_utils_tests/test_supports_tool_choice.py +++ b/tests/litellm_utils_tests/test_supports_tool_choice.py @@ -117,7 +117,9 @@ async def test_supports_tool_choice(): """ # Load model prices litellm._turn_on_debug() - with open("./model_prices_and_context_window.json", "r") as f: + local_path = "../../model_prices_and_context_window.json" + prod_path = "./model_prices_and_context_window.json" + with open(local_path, "r") as f: model_prices = json.load(f) litellm.model_cost = model_prices config_manager = ProviderConfigManager() @@ -133,6 +135,8 @@ async def test_supports_tool_choice(): or any(provider in model_name for provider in OLD_PROVIDERS) or model_info["litellm_provider"] in OLD_PROVIDERS or model_name in block_list + or "azure/eu" in model_name + or "azure/us" in model_name ): continue From 84514f0397e4033d9e3fe428780b653448d8e5f1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 12 Mar 2025 16:51:01 -0700 Subject: [PATCH 10/10] test: fix test --- tests/litellm_utils_tests/test_supports_tool_choice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/litellm_utils_tests/test_supports_tool_choice.py b/tests/litellm_utils_tests/test_supports_tool_choice.py index a68aea4f5a..cfa190f74b 100644 --- a/tests/litellm_utils_tests/test_supports_tool_choice.py +++ b/tests/litellm_utils_tests/test_supports_tool_choice.py @@ -119,7 +119,7 @@ async def test_supports_tool_choice(): litellm._turn_on_debug() local_path = "../../model_prices_and_context_window.json" prod_path = "./model_prices_and_context_window.json" - with open(local_path, "r") as f: + with open(prod_path, "r") as f: model_prices = json.load(f) litellm.model_cost = model_prices config_manager = ProviderConfigManager()