diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 20e0d412ed..a75ecd8cc7 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -218,22 +218,47 @@ class GroqChatConfig(OpenAILikeChatConfig): When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode - You usually want to provide a single tool - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool - - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. + - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model's perspective. + + Note: This workaround is only for models that don't support native json_schema. + Models like gpt-oss-120b, llama-4, kimi-k2 support native json_schema and should + pass response_format directly to Groq. + See: https://console.groq.com/docs/structured-outputs#supported-models """ if json_schema is not None: - _tool_choice = { - "type": "function", - "function": {"name": "json_tool_call"}, - } - _tool = self._create_json_tool_call_for_response_format( - json_schema=json_schema, - ) - optional_params["tools"] = [_tool] - optional_params["tool_choice"] = _tool_choice - optional_params["json_mode"] = True - non_default_params.pop( - "response_format", None - ) # only remove if it's a json_schema - handled via using groq's tool calling params. + # Check if model supports native response_schema + if not litellm.supports_response_schema( + model=model, custom_llm_provider="groq" + ): + # Check if user is also passing tools - this combination won't work + # See: https://console.groq.com/docs/structured-outputs + # "Streaming and tool use are not currently supported with Structured Outputs" + if "tools" in non_default_params: + raise litellm.BadRequestError( + message=f"Groq model '{model}' does not support native structured outputs. " + "LiteLLM uses a tool-calling workaround for structured outputs on this model, " + "which is incompatible with user-provided tools. " + "Either use a model that supports native structured outputs " + "(e.g., gpt-oss-120b, llama-4, kimi-k2), or remove the tools parameter. " + "See: https://console.groq.com/docs/structured-outputs#supported-models", + model=model, + llm_provider="groq", + ) + # Use workaround only for models without native support + _tool_choice = { + "type": "function", + "function": {"name": "json_tool_call"}, + } + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, + ) + optional_params["tools"] = [_tool] + optional_params["tool_choice"] = _tool_choice + optional_params["json_mode"] = True + non_default_params.pop( + "response_format", None + ) # only remove if it's a json_schema - handled via using groq's tool calling params. + # else: model supports native json_schema, let response_format pass through optional_params = super().map_openai_params( non_default_params, optional_params, model, drop_params ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dbaa60e0f1..903b86484c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16881,7 +16881,7 @@ "output_cost_per_token": 9.9e-07, "supports_function_calling": true, "supports_reasoning": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/distil-whisper-large-v3-en": { @@ -16900,7 +16900,7 @@ "mode": "chat", "output_cost_per_token": 7e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/gemma2-9b-it": { @@ -16912,7 +16912,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": false }, "groq/llama-3.1-405b-reasoning": { @@ -16924,7 +16924,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.1-70b-versatile": { @@ -16937,7 +16937,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.1-8b-instant": { @@ -16949,7 +16949,7 @@ "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-11b-text-preview": { @@ -16962,7 +16962,7 @@ "mode": "chat", "output_cost_per_token": 1.8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-11b-vision-preview": { @@ -16975,7 +16975,7 @@ "mode": "chat", "output_cost_per_token": 1.8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true, "supports_vision": true }, @@ -16989,7 +16989,7 @@ "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-3b-preview": { @@ -17002,7 +17002,7 @@ "mode": "chat", "output_cost_per_token": 6e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-90b-text-preview": { @@ -17015,7 +17015,7 @@ "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-90b-vision-preview": { @@ -17028,7 +17028,7 @@ "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true, "supports_vision": true }, @@ -17052,7 +17052,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-guard-3-8b": { @@ -17073,7 +17073,7 @@ "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama3-groq-70b-8192-tool-use-preview": { @@ -17086,7 +17086,7 @@ "mode": "chat", "output_cost_per_token": 8.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama3-groq-8b-8192-tool-use-preview": { @@ -17099,7 +17099,7 @@ "mode": "chat", "output_cost_per_token": 1.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { @@ -17145,7 +17145,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/moonshotai/kimi-k2-instruct": { @@ -17221,7 +17221,7 @@ "output_cost_per_token": 5.9e-07, "supports_function_calling": true, "supports_reasoning": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/whisper-large-v3": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dbaa60e0f1..903b86484c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16881,7 +16881,7 @@ "output_cost_per_token": 9.9e-07, "supports_function_calling": true, "supports_reasoning": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/distil-whisper-large-v3-en": { @@ -16900,7 +16900,7 @@ "mode": "chat", "output_cost_per_token": 7e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/gemma2-9b-it": { @@ -16912,7 +16912,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": false }, "groq/llama-3.1-405b-reasoning": { @@ -16924,7 +16924,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.1-70b-versatile": { @@ -16937,7 +16937,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.1-8b-instant": { @@ -16949,7 +16949,7 @@ "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-11b-text-preview": { @@ -16962,7 +16962,7 @@ "mode": "chat", "output_cost_per_token": 1.8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-11b-vision-preview": { @@ -16975,7 +16975,7 @@ "mode": "chat", "output_cost_per_token": 1.8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true, "supports_vision": true }, @@ -16989,7 +16989,7 @@ "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-3b-preview": { @@ -17002,7 +17002,7 @@ "mode": "chat", "output_cost_per_token": 6e-08, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-90b-text-preview": { @@ -17015,7 +17015,7 @@ "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-3.2-90b-vision-preview": { @@ -17028,7 +17028,7 @@ "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true, "supports_vision": true }, @@ -17052,7 +17052,7 @@ "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama-guard-3-8b": { @@ -17073,7 +17073,7 @@ "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama3-groq-70b-8192-tool-use-preview": { @@ -17086,7 +17086,7 @@ "mode": "chat", "output_cost_per_token": 8.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/llama3-groq-8b-8192-tool-use-preview": { @@ -17099,7 +17099,7 @@ "mode": "chat", "output_cost_per_token": 1.9e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { @@ -17145,7 +17145,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/moonshotai/kimi-k2-instruct": { @@ -17221,7 +17221,7 @@ "output_cost_per_token": 5.9e-07, "supports_function_calling": true, "supports_reasoning": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_tool_choice": true }, "groq/whisper-large-v3": { diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py index fc84580404..33c1a42587 100644 --- a/tests/llm_translation/test_groq.py +++ b/tests/llm_translation/test_groq.py @@ -9,6 +9,7 @@ import pytest # ) # noqa # ) # Adds the parent directory to the system path +import litellm from base_llm_unit_tests import BaseLLMChatTest from litellm.llms.groq.chat.transformation import GroqChatConfig @@ -30,3 +31,136 @@ class TestGroq(BaseLLMChatTest): """Test that reasoning_effort is in the list of supported parameters for Groq""" supported_params = GroqChatConfig().get_supported_openai_params(model=model) assert "reasoning_effort" in supported_params + + +class TestGroqStructuredOutputs: + """ + Tests for Groq structured outputs handling. + Related issues: + - https://github.com/BerriAI/litellm/issues/11001 + - https://github.com/openai/openai-agents-python/issues/2140 + """ + + def test_structured_output_with_tools_raises_error_for_non_native_models(self): + """ + Test that using structured outputs + tools with models that don't support + native json_schema raises a clear error message. + + Groq does not support structured outputs + tools together. + See: https://console.groq.com/docs/structured-outputs + "Streaming and tool use are not currently supported with Structured Outputs" + """ + config = GroqChatConfig() + + # Model that doesn't support native json_schema + model = "llama-3.3-70b-versatile" + + non_default_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test", + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"] + } + } + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}} + } + } + ] + } + + with pytest.raises(litellm.BadRequestError) as exc_info: + config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=False, + ) + + assert "does not support native structured outputs" in str(exc_info.value) + assert "incompatible with user-provided tools" in str(exc_info.value) + + def test_structured_output_without_tools_uses_workaround_for_non_native_models(self): + """ + Test that structured outputs without tools works using the json_tool_call workaround + for models that don't support native json_schema. + """ + config = GroqChatConfig() + + model = "llama-3.3-70b-versatile" + + non_default_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test", + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"] + } + } + } + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=False, + ) + + # Should use the workaround (json_tool_call) + assert "tools" in result + assert len(result["tools"]) == 1 + assert result["tools"][0]["function"]["name"] == "json_tool_call" + assert result["tool_choice"]["function"]["name"] == "json_tool_call" + assert result.get("json_mode") is True + + def test_structured_output_passes_through_for_native_models(self): + """ + Test that structured outputs pass through directly for models that + support native json_schema (e.g., gpt-oss-120b). + """ + config = GroqChatConfig() + + # Model that supports native json_schema + model = "openai/gpt-oss-120b" + + non_default_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test", + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"] + } + } + } + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=False, + ) + + # Should NOT use the workaround - response_format should pass through + # The workaround sets json_mode=True, so if it's not set, we know it passed through + assert result.get("json_mode") is not True + # Should not have the json_tool_call tool + if "tools" in result: + tool_names = [t.get("function", {}).get("name") for t in result["tools"]] + assert "json_tool_call" not in tool_names