From 2331fb45d56d8df47c128526d84e08bf716593dd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Sep 2025 16:53:20 -0700 Subject: [PATCH] =?UTF-8?q?[Bug]:=20Gemini=202.5=20Pro=20=E2=80=93=20schem?= =?UTF-8?q?a=20validation=20fails=20with=20OpenAI-style=20type=20arrays=20?= =?UTF-8?q?in=20tools=20=20(#14154)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: _convert_schema_types * fix recursive detector * test_convert_schema_types_type_array_conversion * fix: DEFAULT_NUM_WORKERS_LITELLM_PROXY --- docs/my-website/docs/proxy/config_settings.md | 1 + litellm/llms/vertex_ai/common_utils.py | 43 +++++ ...odel_prices_and_context_window_backup.json | 156 +++++++++++++++--- .../code_coverage_tests/recursive_detector.py | 3 +- tests/llm_translation/base_llm_unit_tests.py | 46 ++++++ .../vertex_ai/test_vertex_ai_common_utils.py | 65 ++++++++ 6 files changed, 292 insertions(+), 22 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index b03d7ab032..5e20322a8c 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -431,6 +431,7 @@ router_settings: | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 +| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available** | | DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7 | DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03 | DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0 diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 6def8faffe..6c8eb887eb 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -215,6 +215,8 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): # * https://github.com/pydantic/pydantic/discussions/4872 convert_anyof_null_to_nullable(parameters) + _convert_schema_types(parameters) + # Handle empty items objects process_items(parameters) add_object_type(parameters) @@ -439,6 +441,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int: return int(dt.timestamp()) +def _convert_schema_types(schema, depth=0): + """ + Convert type arrays and lowercase types for Vertex AI compatibility. + + Transforms OpenAI-style schemas to Vertex AI format by converting type arrays + like ["string", "number"] to anyOf format and converting all types to uppercase. + """ + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError( + f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting." + ) + + if not isinstance(schema, dict): + return + + + # Handle type field + if "type" in schema: + type_val = schema["type"] + if isinstance(type_val, list) and len(type_val) > 1: + # Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]} + schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)] + schema.pop("type") + elif isinstance(type_val, list) and len(type_val) == 1: + schema["type"] = type_val[0] + elif isinstance(type_val, str): + schema["type"] = type_val + + # Recursively process nested properties, items, and anyOf + for key in ["properties", "items", "anyOf"]: + if key in schema: + value = schema[key] + if key == "properties" and isinstance(value, dict): + for prop_schema in value.values(): + _convert_schema_types(prop_schema, depth + 1) + elif key == "items": + _convert_schema_types(value, depth + 1) + elif key == "anyOf" and isinstance(value, list): + for anyof_schema in value: + _convert_schema_types(anyof_schema, depth + 1) + def get_vertex_project_id_from_url(url: str) -> Optional[str]: """ Get the vertex project id from the url diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4fb87dc186..9abe3dc48f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11991,6 +11991,108 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-4.1": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-5-mini": { "max_tokens": 128000, "max_input_tokens": 400000, @@ -14970,10 +15072,10 @@ "output_cost_per_token": 6e-06, "max_input_tokens": 262000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14981,10 +15083,10 @@ "output_cost_per_token": 2e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14992,10 +15094,10 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -15038,10 +15140,10 @@ "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { @@ -15066,9 +15168,9 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-120b" }, @@ -15077,9 +15179,9 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-20b" }, @@ -15088,12 +15190,24 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/glm-4-5-air" }, + "together_ai/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 1.7e-06, + "max_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "mode": "chat", + "supports_tool_choice": true, + "source": "https://www.together.ai/models/deepseek-v3-1" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 158399305b..c75f95b267 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -25,7 +25,8 @@ IGNORE_FUNCTIONS = [ "filter_value_from_dict", # max depth set. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. - "_remove_json_schema_refs", # max depth set. + "_remove_json_schema_refs", # max depth set., + "_convert_schema_types", # max depth set., ] diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index d36ed94429..f2f785629c 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -141,6 +141,52 @@ class BaseLLMChatTest(ABC): # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None assert response.choices[0].message.content is not None + + def test_tool_call_with_property_type_array(self): + litellm._turn_on_debug() + from litellm.utils import supports_function_calling + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + base_completion_call_args = self.get_base_completion_call_args() + if not supports_function_calling(base_completion_call_args["model"], None): + print("Model does not support function calling") + pytest.skip("Model does not support function calling") + base_completion_call_args = self.get_base_completion_call_args() + response = self.completion_function( + **base_completion_call_args, + messages = [ + { + "role": "user", + "content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike." + } + ], + tools = [ + { + "type": "function", + "function": { + "name": "shoe_get_id", + "description": "Get information about a show by its ID or name", + "parameters": { + "type": "object", + "properties": { + "shoe_id": { + "type": ["string", "number"], + "description": "The shoe ID or name" + } + }, + "required": ["shoe_id"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + } + }, + ] + ) + print(response) + print(json.dumps(response, indent=4, default=str)) + + def test_streaming(self): """Check if litellm handles streaming correctly""" from litellm.types.utils import ModelResponseStream diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 9ebb9004b7..18fffbed59 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -677,3 +677,68 @@ def test_vertex_filter_format_uri(): ) assert "uri" not in json.dumps(new_parameters) + +def test_convert_schema_types_type_array_conversion(): + """ + Test _convert_schema_types function handles type arrays and case conversion. + + This test verifies the fix for the issue where type arrays like ["string", "number"] + would raise an exception in Vertex AI schema validation. + + Relevant issue: https://github.com/BerriAI/litellm/issues/14091 + """ + from litellm.llms.vertex_ai.common_utils import _convert_schema_types + + # Input: OpenAI-style schema with type array (the problematic case) + input_schema = { + "type": "object", + "properties": { + "studio": { + "type": ["string", "number"], + "description": "The studio ID or name" + } + }, + "required": ["studio"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + + # Expected output: Vertex AI compatible schema with anyOf and uppercase types + expected_output = { + "type": "object", + "properties": { + "studio": { + "anyOf": [ + {"type": "string"}, + {"type": "number"} + ], + "description": "The studio ID or name" + } + }, + "required": ["studio"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + + # Apply the transformation + _convert_schema_types(input_schema) + + # Verify the transformation + assert input_schema == expected_output + + # Verify specific transformations: + # 1. Root level type converted to uppercase + assert input_schema["type"] == "object" + + # 2. Type array converted to anyOf format + assert "anyOf" in input_schema["properties"]["studio"] + assert "type" not in input_schema["properties"]["studio"] + + # 3. Individual types in anyOf are uppercase + anyof_types = input_schema["properties"]["studio"]["anyOf"] + assert anyof_types[0]["type"] == "string" + assert anyof_types[1]["type"] == "number" + + # 4. Other properties preserved + assert input_schema["properties"]["studio"]["description"] == "The studio ID or name" + assert input_schema["required"] == ["studio"] \ No newline at end of file