diff --git a/docs/my-website/docs/completion/drop_params.md b/docs/my-website/docs/completion/drop_params.md index e79a88e14b..590d9a4595 100644 --- a/docs/my-website/docs/completion/drop_params.md +++ b/docs/my-website/docs/completion/drop_params.md @@ -107,4 +107,76 @@ response = litellm.completion( -**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model. \ No newline at end of file +**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model. + +## Specify allowed openai params in a request + +Tell litellm to allow specific openai params in a request. Use this if you get a `litellm.UnsupportedParamsError` and want to allow a param. LiteLLM will pass the param as is to the model. + + + + + + +In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param. + +```python showLineNumbers title="Pass allowed_openai_params to LiteLLM Python SDK" +await litellm.acompletion( + model="azure/o_series/", + api_key="xxxxx", + api_base=api_base, + messages=[{"role": "user", "content": "Hello! return a json object"}], + tools=[{"type": "function", "function": {"name": "get_current_time", "description": "Get the current time in a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name, e.g. San Francisco"}}, "required": ["location"]}}}] + allowed_openai_params=["tools"], +) +``` + + + +When using litellm proxy you can pass `allowed_openai_params` in two ways: + +1. Dynamically pass `allowed_openai_params` in a request +2. Set `allowed_openai_params` on the config.yaml file for a specific model + +#### Dynamically pass allowed_openai_params in a request +In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param for a request sent to the model set on the proxy. + +```python showLineNumbers title="Dynamically pass allowed_openai_params in a request" +import openai +from openai import AsyncAzureOpenAI + +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "allowed_openai_params": ["tools"] + } +) +``` + +#### Set allowed_openai_params on config.yaml + +You can also set `allowed_openai_params` on the config.yaml file for a specific model. This means that all requests to this deployment are allowed to pass in the `tools` param. + +```yaml showLineNumbers title="Set allowed_openai_params on config.yaml" +model_list: + - model_name: azure-o1-preview + litellm_params: + model: azure/o_series/ + api_key: xxxxx + api_base: https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview + allowed_openai_params: ["tools"] +``` + + \ No newline at end of file diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 0ca3a28d23..21aafce7fb 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -14,6 +14,7 @@ Translations handled by LiteLLM: from typing import List, Optional +import litellm from litellm import verbose_logger from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_model_info @@ -22,6 +23,27 @@ from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig class AzureOpenAIO1Config(OpenAIOSeriesConfig): + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for the Azure O-Series models + """ + all_openai_params = litellm.OpenAIGPTConfig().get_supported_openai_params( + model=model + ) + non_supported_params = [ + "logprobs", + "top_p", + "presence_penalty", + "frequency_penalty", + "top_logprobs", + ] + + o_series_only_param = ["reasoning_effort"] + all_openai_params.extend(o_series_only_param) + return [ + param for param in all_openai_params if param not in non_supported_params + ] + def should_fake_stream( self, model: Optional[str], diff --git a/litellm/main.py b/litellm/main.py index f69454aaad..56b0aa3671 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1115,6 +1115,7 @@ def completion( # type: ignore # noqa: PLR0915 messages=messages, reasoning_effort=reasoning_effort, thinking=thinking, + allowed_openai_params=kwargs.get("allowed_openai_params"), **non_default_params, ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8716779d1f..51a6ed17b1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1950,6 +1950,7 @@ all_litellm_params = [ "use_in_pass_through", "merge_reasoning_content_in_choices", "litellm_credential_name", + "allowed_openai_params", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) diff --git a/litellm/utils.py b/litellm/utils.py index 1a35e58dc1..4283cf2df1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2843,6 +2843,7 @@ def get_optional_params( # noqa: PLR0915 api_version=None, parallel_tool_calls=None, drop_params=None, + allowed_openai_params: Optional[List[str]] = None, reasoning_effort=None, additional_drop_params=None, messages: Optional[List[AllMessageValues]] = None, @@ -2928,6 +2929,7 @@ def get_optional_params( # noqa: PLR0915 "api_version": None, "parallel_tool_calls": None, "drop_params": None, + "allowed_openai_params": None, "additional_drop_params": None, "messages": None, "reasoning_effort": None, @@ -2944,6 +2946,7 @@ def get_optional_params( # noqa: PLR0915 and k != "custom_llm_provider" and k != "api_version" and k != "drop_params" + and k != "allowed_openai_params" and k != "additional_drop_params" and k != "messages" and k in default_params @@ -3053,6 +3056,12 @@ def get_optional_params( # noqa: PLR0915 tool_function["parameters"] = new_parameters def _check_valid_arg(supported_params: List[str]): + """ + Check if the params passed to completion() are supported by the provider + + Args: + supported_params: List[str] - supported params from the litellm config + """ verbose_logger.info( f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}" ) @@ -3086,7 +3095,7 @@ def get_optional_params( # noqa: PLR0915 else: raise UnsupportedParamsError( status_code=500, - message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", + message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) supported_params = get_supported_openai_params( @@ -3096,7 +3105,14 @@ def get_optional_params( # noqa: PLR0915 supported_params = get_supported_openai_params( model=model, custom_llm_provider="openai" ) - _check_valid_arg(supported_params=supported_params or []) + + supported_params = supported_params or [] + allowed_openai_params = allowed_openai_params or [] + supported_params.extend(allowed_openai_params) + + _check_valid_arg( + supported_params=supported_params or [], + ) ## raise exception if provider doesn't support passed in param if custom_llm_provider == "anthropic": ## check if unsupported param passed in @@ -3735,6 +3751,26 @@ def get_optional_params( # noqa: PLR0915 if k not in default_params.keys(): optional_params[k] = passed_params[k] print_verbose(f"Final returned optional params: {optional_params}") + optional_params = _apply_openai_param_overrides( + optional_params=optional_params, + non_default_params=non_default_params, + allowed_openai_params=allowed_openai_params, + ) + return optional_params + + +def _apply_openai_param_overrides( + optional_params: dict, non_default_params: dict, allowed_openai_params: list +): + """ + If user passes in allowed_openai_params, apply them to optional_params + + These params will get passed as is to the LLM API since the user opted in to passing them in the request + """ + if allowed_openai_params: + for param in allowed_openai_params: + if param not in optional_params: + optional_params[param] = non_default_params.pop(param, None) return optional_params diff --git a/tests/litellm_utils_tests/test_supports_tool_choice.py b/tests/litellm_utils_tests/test_supports_tool_choice.py index cfa190f74b..e09be15319 100644 --- a/tests/litellm_utils_tests/test_supports_tool_choice.py +++ b/tests/litellm_utils_tests/test_supports_tool_choice.py @@ -137,6 +137,8 @@ async def test_supports_tool_choice(): or model_name in block_list or "azure/eu" in model_name or "azure/us" in model_name + or "o1" in model_name + or "o3" in model_name ): continue diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index 13ba4169ce..3d3764f049 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -21,9 +21,10 @@ from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest): def get_base_completion_call_args(self): return { - "model": "azure/o1-preview", + "model": "azure/o1", "api_key": os.getenv("AZURE_OPENAI_O1_KEY"), - "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com", + "api_base": "https://openai-prod-test.openai.azure.com", + "api_version": "2024-12-01-preview" } def get_client(self): @@ -31,7 +32,7 @@ class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest): return AzureOpenAI( api_key="my-fake-o1-key", - base_url="https://openai-gpt-4-test-v-1.openai.azure.com", + base_url="https://openai-prod-test.openai.azure.com", api_version="2024-02-15-preview", ) @@ -170,3 +171,54 @@ def test_openai_o_series_max_retries_0(mock_get_openai_client): mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + + +@pytest.mark.asyncio +async def test_azure_o1_series_response_format_extra_params(): + """ + Tool calling should work for all azure o_series models. + """ + litellm._turn_on_debug() + + from openai import AsyncAzureOpenAI + + litellm.set_verbose = True + + client = AsyncAzureOpenAI( + api_key="fake-api-key", + base_url="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview", + api_version="2025-01-01-preview" + ) + + tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + response_format = {'type': 'json_object'} + tool_choice = "auto" + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + await litellm.acompletion( + client=client, + model="azure/o_series/", + api_key="xxxxx", + api_base="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview", + api_version="2024-12-01-preview", + messages=[{"role": "user", "content": "Hello! return a json object"}], + tools=tools, + response_format=response_format, + tool_choice=tool_choice + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + print("request_body: ", json.dumps(request_body, indent=4)) + assert request_body["tools"] == tools + assert request_body["response_format"] == response_format + assert request_body["tool_choice"] == tool_choice + + + + diff --git a/tests/llm_translation/test_cohere.py b/tests/llm_translation/test_cohere.py index 124a5c8788..6b4d3a2045 100644 --- a/tests/llm_translation/test_cohere.py +++ b/tests/llm_translation/test_cohere.py @@ -17,6 +17,9 @@ import pytest import litellm from litellm import RateLimitError, Timeout, completion, completion_cost, embedding +from unittest.mock import AsyncMock, patch +from litellm import RateLimitError, Timeout, completion, completion_cost, embedding +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler litellm.num_retries = 3 @@ -224,3 +227,57 @@ async def test_chat_completion_cohere_stream(sync_mode): pass except Exception as e: pytest.fail(f"Error occurred: {e}") + + +@pytest.mark.asyncio +async def test_cohere_request_body_with_allowed_params(): + """ + Test to validate that when allowed_openai_params is provided, the request body contains + the correct response_format and reasoning_effort values. + """ + # Define test parameters + test_response_format = {"type": "json"} + test_reasoning_effort = "low" + test_tools = [{ + "type": "function", + "function": { + "name": "get_current_time", + "description": "Get the current time in a given location.", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name, e.g. San Francisco"} + }, + "required": ["location"] + } + } + }] + + client = AsyncHTTPHandler() + + # Mock the post method + with patch.object(client, "post", new=AsyncMock()) as mock_post: + try: + await litellm.acompletion( + model="cohere/command", + messages=[{"content": "what llm are you", "role": "user"}], + allowed_openai_params=["tools", "response_format", "reasoning_effort"], + response_format=test_response_format, + reasoning_effort=test_reasoning_effort, + tools=test_tools, + client=client + ) + except Exception: + pass # We only care about the request body validation + + # Verify the API call was made + mock_post.assert_called_once() + + # Get and parse the request body + request_data = json.loads(mock_post.call_args.kwargs["data"]) + print(f"request_data: {request_data}") + + # Validate request contains our specified parameters + assert "allowed_openai_params" not in request_data + assert request_data["response_format"] == test_response_format + assert request_data["reasoning_effort"] == test_reasoning_effort diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 45dc3d8f6f..5e792d46e9 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -67,6 +67,30 @@ def test_anthropic_optional_params(stop_sequence, expected_count): assert len(optional_params) == expected_count + + +def test_get_optional_params_with_allowed_openai_params(): + """ + Test if use can dynamically pass in allowed_openai_params to override default behavior + """ + litellm.drop_params = True + tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + response_format = {"type": "json"} + reasoning_effort = "low" + optional_params = get_optional_params( + model="cf/llama-3.1-70b-instruct", + custom_llm_provider="cloudflare", + allowed_openai_params=["tools", "reasoning_effort", "response_format"], + tools=tools, + response_format=response_format, + reasoning_effort=reasoning_effort, + ) + print(f"optional_params: {optional_params}") + assert optional_params["tools"] == tools + assert optional_params["response_format"] == response_format + assert optional_params["reasoning_effort"] == reasoning_effort + + def test_bedrock_optional_params_embeddings(): litellm.drop_params = True optional_params = get_optional_params_embeddings( @@ -1380,6 +1404,16 @@ def test_azure_modalities_param(): assert optional_params["modalities"] == ["text", "audio"] assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"} + + +def test_azure_response_format_param(): + optional_params = litellm.get_optional_params( + model="azure/o_series/test-o3-mini", + custom_llm_provider="azure/o_series", + tools= [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + ) + + @pytest.mark.parametrize( "model, provider", [ @@ -1396,3 +1430,4 @@ def test_anthropic_unified_reasoning_content(model, provider): reasoning_effort="high", ) assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096} + diff --git a/tests/local_testing/test_bad_params.py b/tests/local_testing/test_bad_params.py index ef3b4596ec..221135df90 100644 --- a/tests/local_testing/test_bad_params.py +++ b/tests/local_testing/test_bad_params.py @@ -44,7 +44,7 @@ def test_completion_invalid_param_cohere(): except Exception as e: assert isinstance(e, litellm.UnsupportedParamsError) print("got an exception=", str(e)) - if " cohere does not support parameters: {'seed': 12}" in str(e): + if "cohere does not support parameters: ['seed']" in str(e): pass else: pytest.fail(f"An error occurred {e}")