Merge pull request #9694 from BerriAI/litellm_fix_azure_o_series

[Bug fix] Azure o-series tool calling

docs/my-website/docs/completion/drop_params.md

@@ -107,4 +107,76 @@ response = litellm.completion(
 </TabItem>
 </Tabs>
 
-**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model.
+**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model.
+
+## Specify allowed openai params in a request
+
+Tell litellm to allow specific openai params in a request. Use this if you get a `litellm.UnsupportedParamsError` and want to allow a param. LiteLLM will pass the param as is to the model.
+
+
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param.
+
+```python showLineNumbers title="Pass allowed_openai_params to LiteLLM Python SDK"
+await litellm.acompletion(
+    model="azure/o_series/<my-deployment-name>",
+    api_key="xxxxx",
+    api_base=api_base,
+    messages=[{"role": "user", "content": "Hello! return a json object"}],
+    tools=[{"type": "function", "function": {"name": "get_current_time", "description": "Get the current time in a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name, e.g. San Francisco"}}, "required": ["location"]}}}]
+    allowed_openai_params=["tools"],
+)
+```
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+When using litellm proxy you can pass `allowed_openai_params` in two ways:
+
+1. Dynamically pass `allowed_openai_params` in a request
+2. Set `allowed_openai_params` on the config.yaml file for a specific model
+
+#### Dynamically pass allowed_openai_params in a request
+In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param for a request sent to the model set on the proxy.
+
+```python showLineNumbers title="Dynamically pass allowed_openai_params in a request"
+import openai
+from openai import AsyncAzureOpenAI
+
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages = [
+        {
+            "role": "user",
+            "content": "this is a test request, write a short poem"
+        }
+    ],
+    extra_body={ 
+        "allowed_openai_params": ["tools"]
+    }
+)
+```
+
+#### Set allowed_openai_params on config.yaml
+
+You can also set `allowed_openai_params` on the config.yaml file for a specific model. This means that all requests to this deployment are allowed to pass in the `tools` param.
+
+```yaml showLineNumbers title="Set allowed_openai_params on config.yaml"
+model_list:
+  - model_name: azure-o1-preview
+    litellm_params:
+      model: azure/o_series/<my-deployment-name>
+      api_key: xxxxx
+      api_base: https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview
+      allowed_openai_params: ["tools"]
+```
+</TabItem>
+</Tabs>

litellm/llms/azure/chat/o_series_transformation.py

@@ -14,6 +14,7 @@ Translations handled by LiteLLM:
 
 from typing import List, Optional
 
+import litellm
 from litellm import verbose_logger
 from litellm.types.llms.openai import AllMessageValues
 from litellm.utils import get_model_info
@@ -22,6 +23,27 @@ from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig
 
 
 class AzureOpenAIO1Config(OpenAIOSeriesConfig):
+    def get_supported_openai_params(self, model: str) -> list:
+        """
+        Get the supported OpenAI params for the Azure O-Series models
+        """
+        all_openai_params = litellm.OpenAIGPTConfig().get_supported_openai_params(
+            model=model
+        )
+        non_supported_params = [
+            "logprobs",
+            "top_p",
+            "presence_penalty",
+            "frequency_penalty",
+            "top_logprobs",
+        ]
+
+        o_series_only_param = ["reasoning_effort"]
+        all_openai_params.extend(o_series_only_param)
+        return [
+            param for param in all_openai_params if param not in non_supported_params
+        ]
+
     def should_fake_stream(
         self,
         model: Optional[str],

litellm/main.py

@@ -1115,6 +1115,7 @@ def completion(  # type: ignore # noqa: PLR0915
             messages=messages,
             reasoning_effort=reasoning_effort,
             thinking=thinking,
+            allowed_openai_params=kwargs.get("allowed_openai_params"),
             **non_default_params,
         )
 

litellm/types/utils.py

@@ -1950,6 +1950,7 @@ all_litellm_params = [
     "use_in_pass_through",
     "merge_reasoning_content_in_choices",
     "litellm_credential_name",
+    "allowed_openai_params",
 ] + list(StandardCallbackDynamicParams.__annotations__.keys())
 
 

litellm/utils.py

@@ -2843,6 +2843,7 @@ def get_optional_params(  # noqa: PLR0915
     api_version=None,
     parallel_tool_calls=None,
     drop_params=None,
+    allowed_openai_params: Optional[List[str]] = None,
     reasoning_effort=None,
     additional_drop_params=None,
     messages: Optional[List[AllMessageValues]] = None,
@@ -2928,6 +2929,7 @@ def get_optional_params(  # noqa: PLR0915
         "api_version": None,
         "parallel_tool_calls": None,
         "drop_params": None,
+        "allowed_openai_params": None,
         "additional_drop_params": None,
         "messages": None,
         "reasoning_effort": None,
@@ -2944,6 +2946,7 @@ def get_optional_params(  # noqa: PLR0915
             and k != "custom_llm_provider"
             and k != "api_version"
             and k != "drop_params"
+            and k != "allowed_openai_params"
             and k != "additional_drop_params"
             and k != "messages"
             and k in default_params
@@ -3053,6 +3056,12 @@ def get_optional_params(  # noqa: PLR0915
                 tool_function["parameters"] = new_parameters
 
     def _check_valid_arg(supported_params: List[str]):
+        """
+        Check if the params passed to completion() are supported by the provider
+
+        Args:
+            supported_params: List[str] - supported params from the litellm config
+        """
         verbose_logger.info(
             f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}"
         )
@@ -3086,7 +3095,7 @@ def get_optional_params(  # noqa: PLR0915
             else:
                 raise UnsupportedParamsError(
                     status_code=500,
-                    message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
+                    message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.",
                 )
 
     supported_params = get_supported_openai_params(
@@ -3096,7 +3105,14 @@ def get_optional_params(  # noqa: PLR0915
         supported_params = get_supported_openai_params(
             model=model, custom_llm_provider="openai"
         )
-    _check_valid_arg(supported_params=supported_params or [])
+
+    supported_params = supported_params or []
+    allowed_openai_params = allowed_openai_params or []
+    supported_params.extend(allowed_openai_params)
+
+    _check_valid_arg(
+        supported_params=supported_params or [],
+    )
     ## raise exception if provider doesn't support passed in param
     if custom_llm_provider == "anthropic":
         ## check if unsupported param passed in
@@ -3735,6 +3751,26 @@ def get_optional_params(  # noqa: PLR0915
             if k not in default_params.keys():
                 optional_params[k] = passed_params[k]
     print_verbose(f"Final returned optional params: {optional_params}")
+    optional_params = _apply_openai_param_overrides(
+        optional_params=optional_params,
+        non_default_params=non_default_params,
+        allowed_openai_params=allowed_openai_params,
+    )
+    return optional_params
+
+
+def _apply_openai_param_overrides(
+    optional_params: dict, non_default_params: dict, allowed_openai_params: list
+):
+    """
+    If user passes in allowed_openai_params, apply them to optional_params
+
+    These params will get passed as is to the LLM API since the user opted in to passing them in the request
+    """
+    if allowed_openai_params:
+        for param in allowed_openai_params:
+            if param not in optional_params:
+                optional_params[param] = non_default_params.pop(param, None)
     return optional_params
 
 

tests/litellm_utils_tests/test_supports_tool_choice.py

@@ -137,6 +137,8 @@ async def test_supports_tool_choice():
             or model_name in block_list
             or "azure/eu" in model_name
             or "azure/us" in model_name
+            or "o1" in model_name
+            or "o3" in model_name
         ):
             continue
 

tests/llm_translation/test_azure_o_series.py

@@ -21,9 +21,10 @@ from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest
 class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest):
     def get_base_completion_call_args(self):
         return {
-            "model": "azure/o1-preview",
+            "model": "azure/o1",
             "api_key": os.getenv("AZURE_OPENAI_O1_KEY"),
-            "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com",
+            "api_base": "https://openai-prod-test.openai.azure.com",
+            "api_version": "2024-12-01-preview"
         }
 
     def get_client(self):
@@ -31,7 +32,7 @@ class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest):
 
         return AzureOpenAI(
             api_key="my-fake-o1-key",
-            base_url="https://openai-gpt-4-test-v-1.openai.azure.com",
+            base_url="https://openai-prod-test.openai.azure.com",
             api_version="2024-02-15-preview",
         )
 
@@ -170,3 +171,54 @@ def test_openai_o_series_max_retries_0(mock_get_openai_client):
 
     mock_get_openai_client.assert_called_once()
     assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0
+
+
+@pytest.mark.asyncio
+async def test_azure_o1_series_response_format_extra_params():
+    """
+    Tool calling should work for all azure o_series models.
+    """
+    litellm._turn_on_debug()
+
+    from openai import AsyncAzureOpenAI
+
+    litellm.set_verbose = True
+
+    client = AsyncAzureOpenAI(
+        api_key="fake-api-key", 
+        base_url="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview", 
+        api_version="2025-01-01-preview"
+    )
+
+    tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}]
+    response_format = {'type': 'json_object'}
+    tool_choice = "auto"
+    with patch.object(
+        client.chat.completions.with_raw_response, "create"
+    ) as mock_client:
+        try:
+            await litellm.acompletion(
+                client=client,
+                model="azure/o_series/<my-deployment-name>",
+                api_key="xxxxx",
+                api_base="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview",
+                api_version="2024-12-01-preview",
+                messages=[{"role": "user", "content": "Hello! return a json object"}],
+                tools=tools,
+                response_format=response_format,
+                tool_choice=tool_choice
+            )
+        except Exception as e:
+            print(f"Error: {e}")
+
+        mock_client.assert_called_once()
+        request_body = mock_client.call_args.kwargs
+
+        print("request_body: ", json.dumps(request_body, indent=4))
+        assert request_body["tools"] == tools
+        assert request_body["response_format"] == response_format
+        assert request_body["tool_choice"] == tool_choice
+
+    
+
+

tests/llm_translation/test_cohere.py

@@ -17,6 +17,9 @@ import pytest
 
 import litellm
 from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
+from unittest.mock import AsyncMock, patch
+from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 
 litellm.num_retries = 3
 
@@ -224,3 +227,57 @@ async def test_chat_completion_cohere_stream(sync_mode):
         pass
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
+
+
+@pytest.mark.asyncio
+async def test_cohere_request_body_with_allowed_params():
+    """
+    Test to validate that when allowed_openai_params is provided, the request body contains
+    the correct response_format and reasoning_effort values.
+    """
+    # Define test parameters
+    test_response_format = {"type": "json"}
+    test_reasoning_effort = "low"
+    test_tools = [{
+        "type": "function", 
+        "function": {
+            "name": "get_current_time", 
+            "description": "Get the current time in a given location.", 
+            "parameters": {
+                "type": "object", 
+                "properties": {
+                    "location": {"type": "string", "description": "The city name, e.g. San Francisco"}
+                },
+                "required": ["location"]
+            }
+        }
+    }]
+
+    client = AsyncHTTPHandler()
+
+    # Mock the post method
+    with patch.object(client, "post", new=AsyncMock()) as mock_post:
+        try:
+            await litellm.acompletion(
+                model="cohere/command",
+                messages=[{"content": "what llm are you", "role": "user"}],
+                allowed_openai_params=["tools", "response_format", "reasoning_effort"],
+                response_format=test_response_format,
+                reasoning_effort=test_reasoning_effort,
+                tools=test_tools,
+                client=client
+            )
+        except Exception:
+            pass  # We only care about the request body validation
+
+        # Verify the API call was made
+        mock_post.assert_called_once()
+        
+        # Get and parse the request body
+        request_data = json.loads(mock_post.call_args.kwargs["data"])
+        print(f"request_data: {request_data}")
+        
+        # Validate request contains our specified parameters
+        assert "allowed_openai_params" not in request_data
+        assert request_data["response_format"] == test_response_format
+        assert request_data["reasoning_effort"] == test_reasoning_effort

tests/llm_translation/test_optional_params.py

@@ -67,6 +67,30 @@ def test_anthropic_optional_params(stop_sequence, expected_count):
     assert len(optional_params) == expected_count
 
 
+
+
+def test_get_optional_params_with_allowed_openai_params():
+    """
+    Test if use can dynamically pass in allowed_openai_params to override default behavior
+    """
+    litellm.drop_params = True
+    tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}]
+    response_format = {"type": "json"}
+    reasoning_effort = "low"
+    optional_params = get_optional_params(
+        model="cf/llama-3.1-70b-instruct", 
+        custom_llm_provider="cloudflare", 
+        allowed_openai_params=["tools", "reasoning_effort", "response_format"],
+        tools=tools,
+        response_format=response_format,
+        reasoning_effort=reasoning_effort,
+    )
+    print(f"optional_params: {optional_params}")
+    assert optional_params["tools"] == tools
+    assert optional_params["response_format"] == response_format
+    assert optional_params["reasoning_effort"] == reasoning_effort  
+
+
 def test_bedrock_optional_params_embeddings():
     litellm.drop_params = True
     optional_params = get_optional_params_embeddings(
@@ -1380,6 +1404,16 @@ def test_azure_modalities_param():
     assert optional_params["modalities"] == ["text", "audio"]
     assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"}
 
+
+
+def test_azure_response_format_param():
+    optional_params = litellm.get_optional_params(
+        model="azure/o_series/test-o3-mini",
+        custom_llm_provider="azure/o_series",
+        tools= [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}]
+    )
+
+    
 @pytest.mark.parametrize(
     "model, provider",
     [
@@ -1396,3 +1430,4 @@ def test_anthropic_unified_reasoning_content(model, provider):
         reasoning_effort="high",
     )
     assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096}
+

tests/local_testing/test_bad_params.py

@@ -44,7 +44,7 @@ def test_completion_invalid_param_cohere():
     except Exception as e:
         assert isinstance(e, litellm.UnsupportedParamsError)
         print("got an exception=", str(e))
-        if " cohere does not support parameters: {'seed': 12}" in str(e):
+        if "cohere does not support parameters: ['seed']" in str(e):
             pass
         else:
             pytest.fail(f"An error occurred {e}")