From f4e976e22570d23d9aed896d85a8bbcbd6c9b2db Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 22 Apr 2026 09:45:21 +0530 Subject: [PATCH 1/3] fix(anthropic): handle response_format tool alongside user tools in non-streaming Non-streaming path required len(tool_calls)==1 to unwrap json_tool_call, so mixed user tools leaked the internal tool. Align with Bedrock converse handling: strip internal tools, merge structured JSON into content. Made-with: Cursor --- litellm/llms/anthropic/chat/transformation.py | 79 ++++++++++++++----- .../test_anthropic_chat_transformation.py | 34 ++++++++ 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index cd5bb73171..4a6743b74b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1549,25 +1549,56 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) data["output_config"] = output_config + def _resolve_json_mode_non_streaming( + self, + json_mode: Optional[bool], + tool_calls: List[ChatCompletionToolCallChunk], + ) -> Tuple[ + Optional[LitellmMessage], + List[ChatCompletionToolCallChunk], + Optional[str], + ]: + """Strip internal response_format tool calls; merge payload into content when mixed with user tools.""" + if json_mode is not True or not tool_calls: + return None, tool_calls, None + + json_indices = [ + i + for i, t in enumerate(tool_calls) + if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + if not json_indices: + return None, tool_calls, None + + if len(json_indices) == len(tool_calls): + json_tool = tool_calls[json_indices[0]] + if json_tool.get("function", {}).get("arguments") is None: + return None, tool_calls, None + _message = AnthropicConfig._convert_tool_response_to_message( + tool_calls=[json_tool] + ) + return _message, [], None + + first_json = tool_calls[json_indices[0]] + json_msg = AnthropicConfig._convert_tool_response_to_message([first_json]) + extra_content: Optional[str] = ( + json_msg.content if json_msg is not None else None + ) + filtered_tools = [ + t for i, t in enumerate(tool_calls) if i not in json_indices + ] + return None, filtered_tools, extra_content + def _transform_response_for_json_mode( self, json_mode: Optional[bool], tool_calls: List[ChatCompletionToolCallChunk], ) -> Optional[LitellmMessage]: - _message: Optional[LitellmMessage] = None - if json_mode is True and len(tool_calls) == 1: - # check if tool name is the default tool name - json_mode_content_str: Optional[str] = None - if ( - "name" in tool_calls[0]["function"] - and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME - ): - json_mode_content_str = tool_calls[0]["function"].get("arguments") - if json_mode_content_str is not None: - _message = AnthropicConfig._convert_tool_response_to_message( - tool_calls=tool_calls, - ) - return _message + replacement, _, _ = self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + return replacement def extract_response_content(self, completion_response: dict) -> Tuple[ str, @@ -1927,19 +1958,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_calls, ) + json_mode_message, tool_calls_for_message, json_extra_content = ( + self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + ) + merged_text = text_content or "" + if json_extra_content: + merged_text = ( + merged_text + json_extra_content if merged_text else json_extra_content + ) + _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, + tool_calls=tool_calls_for_message, + content=merged_text or None, provider_specific_fields=provider_specific_fields, thinking_blocks=thinking_blocks, reasoning_content=reasoning_content, ) _message.provider_specific_fields = provider_specific_fields - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) if json_mode_message is not None: completion_response["stop_reason"] = "stop" _message = json_mode_message diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a7f5f92ab0..84a985b275 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -8,6 +8,7 @@ sys.path.insert( ) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -38,6 +39,39 @@ def test_response_format_transformation_unit_test(): print(result) +def test_anthropic_json_mode_non_streaming_mixed_internal_and_user_tools(): + """Non-streaming + response_format: internal json tool must not require len(tool_calls)==1.""" + config = AnthropicConfig() + tool_calls = [ + { + "id": "toolu_json", + "type": "function", + "function": { + "name": RESPONSE_FORMAT_TOOL_NAME, + "arguments": '{"values": {"answer": 42}}', + }, + "index": 0, + }, + { + "id": "toolu_user", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NY"}', + }, + "index": 1, + }, + ] + replacement, filtered, extra = config._resolve_json_mode_non_streaming( + json_mode=True, + tool_calls=tool_calls, + ) + assert replacement is None + assert len(filtered) == 1 + assert filtered[0]["function"]["name"] == "get_weather" + assert extra == '{"answer": 42}' + + def test_calculate_usage(): """ Do not include cache_creation_input_tokens in the prompt_tokens From f503c061a59ed329de1967914e4e55124d352b8b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 24 Apr 2026 09:11:55 +0530 Subject: [PATCH 2/3] Fix black formatting --- litellm/llms/anthropic/chat/transformation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 4a6743b74b..9aaaa39fd7 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1584,9 +1584,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): extra_content: Optional[str] = ( json_msg.content if json_msg is not None else None ) - filtered_tools = [ - t for i, t in enumerate(tool_calls) if i not in json_indices - ] + filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices] return None, filtered_tools, extra_content def _transform_response_for_json_mode( From 3bbb5c7fd74d1f4c08abd05c2a8ccf7e5c20d37f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:56:50 +0000 Subject: [PATCH 3/3] Remove dead _transform_response_for_json_mode wrapper The wrapper had no production callers after transform_parsed_response was refactored to call _resolve_json_mode_non_streaming directly. Updated the parametrized test to call the underlying method. --- litellm/llms/anthropic/chat/transformation.py | 11 ----------- tests/llm_translation/test_anthropic_completion.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9aaaa39fd7..2c5d7901ba 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1587,17 +1587,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices] return None, filtered_tools, extra_content - def _transform_response_for_json_mode( - self, - json_mode: Optional[bool], - tool_calls: List[ChatCompletionToolCallChunk], - ) -> Optional[LitellmMessage]: - replacement, _, _ = self._resolve_json_mode_non_streaming( - json_mode=json_mode, - tool_calls=tool_calls, - ) - return replacement - def extract_response_content(self, completion_response: dict) -> Tuple[ str, Optional[List[Any]], diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index fdf8c24ac9..7b2b6bed6a 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -870,7 +870,7 @@ from litellm.constants import RESPONSE_FORMAT_TOOL_NAME def test_anthropic_json_mode_and_tool_call_response( json_mode, tool_calls, expect_null_response ): - result = litellm.AnthropicConfig()._transform_response_for_json_mode( + result, _, _ = litellm.AnthropicConfig()._resolve_json_mode_non_streaming( json_mode=json_mode, tool_calls=tool_calls, )