From ac5a4df72442ef6ee8f13df0c9c0f1ca921fa7f9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 6 Jan 2026 15:17:35 +0530 Subject: [PATCH] Fix: vertex ai doesn't support structured output --- .../anthropic/transformation.py | 34 +++++ ...partner_models_anthropic_transformation.py | 144 ++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 24425f08b5..1df07f405e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -69,6 +69,9 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter + # VertexAI doesn't support output_format parameter, remove it if present + data.pop("output_format", None) + tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) auto_betas = self.get_anthropic_beta_list( @@ -89,6 +92,37 @@ class VertexAIAnthropicConfig(AnthropicConfig): return data + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Override parent method to ensure VertexAI always uses tool-based structured outputs. + VertexAI doesn't support the output_format parameter, so we force all models + to use the tool-based approach for structured outputs. + """ + # Temporarily override model name to force tool-based approach + # This ensures Claude Sonnet 4.5 uses tools instead of output_format + original_model = model + if "response_format" in non_default_params: + model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach + + # Call parent method with potentially modified model name + optional_params = super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + # Restore original model name for any other processing + model = original_model + + return optional_params + def transform_response( self, model: str, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 5f2dd387b9..7b60a0a336 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -115,3 +115,147 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): "Non-Vertex request SHOULD have anthropic-beta header for structured output" assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", \ f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" + + +def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): + """ + Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based + structured outputs instead of output_format parameter. + + This test verifies that: + 1. Claude Sonnet 4.5 uses tool-based structured outputs on VertexAI + 2. output_format parameter is removed from the final request + 3. The fix prevents "Extra inputs are not permitted" error + """ + config = VertexAIAnthropicConfig() + + # Test data matching the issue report + response_format = { + "type": "json_schema", + "json_schema": { + "name": "questions", + "strict": True, + "schema": { + "type": "object", + "properties": { + "question": { + "type": "string" + }, + "response": { + "type": "string" + } + }, + "required": ["question", "response"], + "additionalProperties": False + } + } + } + + messages = [ + {"role": "user", "content": "Generate a question and answer about AI."} + ] + + # Test parameters that would trigger the issue + non_default_params = { + "response_format": response_format, + "max_tokens": 1000, + } + + # Test 1: Verify map_openai_params forces tool-based approach for Claude Sonnet 4.5 + optional_params = {} + result_params = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-3-5-sonnet-20241022", # Claude Sonnet 4.5 model + drop_params=False, + ) + + # Should have tools and tool_choice (tool-based approach) + assert "tools" in result_params, "Tools should be present for structured output" + assert "tool_choice" in result_params, "Tool choice should be present for structured output" + assert "json_mode" in result_params, "JSON mode should be enabled" + + # Verify the tool is the response format tool + tools = result_params["tools"] + assert len(tools) == 1, "Should have exactly one tool for response format" + assert tools[0]["name"] == "json_tool_call", "Tool should be named json_tool_call" + + # Test 2: Verify transform_request removes output_format parameter + # Simulate what would happen if parent class added output_format + test_data = { + "model": "claude-3-5-sonnet-20241022", + "messages": messages, + "max_tokens": 1000, + "tools": tools, + "tool_choice": result_params["tool_choice"], + "output_format": { # This would be added by parent class for Sonnet 4.5 + "type": "json_schema", + "schema": response_format["json_schema"]["schema"] + } + } + + # Mock the parent transform_request to return data with output_format + original_transform = config.__class__.__bases__[0].transform_request + + def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): + # Return test data that includes output_format + return test_data.copy() + + # Temporarily replace parent method + config.__class__.__bases__[0].transform_request = mock_transform_request + + try: + final_data = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=result_params, + litellm_params={}, + headers={}, + ) + + # Verify that output_format was removed (fixes the "Extra inputs are not permitted" error) + assert "output_format" not in final_data, "output_format should be removed for VertexAI" + assert "model" not in final_data, "model should be removed for VertexAI" + assert "tools" in final_data, "tools should still be present" + assert "tool_choice" in final_data, "tool_choice should still be present" + + finally: + # Restore original method + config.__class__.__bases__[0].transform_request = original_transform + + +def test_vertex_ai_anthropic_other_models_still_use_tools(): + """ + Test that other Anthropic models (non-Sonnet 4.5) on VertexAI also use tool-based + structured outputs, ensuring consistency across all models. + """ + config = VertexAIAnthropicConfig() + + response_format = { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": { + "type": "object", + "properties": { + "result": {"type": "string"} + } + } + } + } + + # Test with Claude 3 Sonnet (not 4.5) + non_default_params = {"response_format": response_format} + optional_params = {} + + result_params = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-3-sonnet-20240229", + drop_params=False, + ) + + # Should still use tool-based approach + assert "tools" in result_params, "Claude 3 Sonnet should also use tool-based structured output" + assert "tool_choice" in result_params, "Tool choice should be present" + assert "json_mode" in result_params, "JSON mode should be enabled"