diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 23175c9f2f..04cf66cf02 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -22,6 +22,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -487,6 +488,11 @@ class ModelResponseIterator: self.tool_index = -1 self.json_mode = json_mode + # Track if we're currently streaming a response_format tool + self.is_response_format_tool: bool = False + # Track if we've converted any response_format tools (affects finish_reason) + self.converted_response_format_tool: bool = False + def check_empty_tool_call_args(self) -> bool: """ Check if the tool call block so far has been an empty string @@ -515,7 +521,9 @@ class ModelResponseIterator: usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None ) - def _content_block_delta_helper(self, chunk: dict) -> Tuple[ + def _content_block_delta_helper( + self, chunk: dict + ) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], @@ -680,7 +688,6 @@ class ModelResponseIterator: ContentBlockStop(**chunk) # type: ignore # check if tool call content block is_empty = self.check_empty_tool_call_args() - if is_empty: tool_use = { "id": None, @@ -691,18 +698,10 @@ class ModelResponseIterator: }, "index": self.tool_index, } + # Reset response_format tool tracking when block stops + self.is_response_format_tool = False elif type_chunk == "message_delta": - """ - Anthropic - chunk = {'type': 'message_delta', 'delta': {'stop_reason': 'max_tokens', 'stop_sequence': None}, 'usage': {'output_tokens': 10}} - """ - # TODO - get usage from this chunk, set in response - message_delta = MessageBlockDelta(**chunk) # type: ignore - finish_reason = map_finish_reason( - finish_reason=message_delta["delta"].get("stop_reason", "stop") - or "stop" - ) - usage = self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) + finish_reason, usage = self._handle_message_delta(chunk) elif type_chunk == "message_start": """ Anthropic @@ -778,6 +777,13 @@ class ModelResponseIterator: Anthropic returns the JSON schema as part of the tool call OpenAI returns the JSON schema as part of the content, this handles placing it in the content + Tool streaming follows Anthropic's fine-grained streaming pattern: + - content_block_start: Contains complete tool info (id, name, empty arguments) + - content_block_delta: Contains argument deltas (partial_json) + - content_block_stop: Signals end of tool + + Reference: https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/fine-grained-tool-streaming + Args: text: str tool_use: Optional[ChatCompletionToolCallChunk] @@ -787,16 +793,50 @@ class ModelResponseIterator: text: The text to use in the content tool_use: The ChatCompletionToolCallChunk to use in the chunk response """ - if self.json_mode is True and tool_use is not None: + if not self.json_mode or tool_use is None: + return text, tool_use + + # Check if this is a new tool call (has id) + if tool_use.get("id") is not None: + # New tool call from content_block_start - tool name is always complete here + # (per Anthropic's fine-grained streaming pattern) + tool_name = tool_use.get("function", {}).get("name", "") + self.is_response_format_tool = tool_name == RESPONSE_FORMAT_TOOL_NAME + + # Convert tool to content if we're tracking a response_format tool + if self.is_response_format_tool: message = AnthropicConfig._convert_tool_response_to_message( tool_calls=[tool_use] ) if message is not None: text = message.content or "" tool_use = None + # Track that we converted a response_format tool + self.converted_response_format_tool = True return text, tool_use + def _handle_message_delta(self, chunk: dict) -> Tuple[str, Optional[Usage]]: + """ + Handle message_delta event for finish_reason and usage. + + Args: + chunk: The message_delta chunk + + Returns: + Tuple of (finish_reason, usage) + """ + message_delta = MessageBlockDelta(**chunk) # type: ignore + finish_reason = map_finish_reason( + finish_reason=message_delta["delta"].get("stop_reason", "stop") or "stop" + ) + # Override finish_reason to "stop" if we converted response_format tools + # (matches OpenAI behavior and non-streaming Anthropic implementation) + if self.converted_response_format_tool: + finish_reason = "stop" + usage = self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) + return finish_reason, usage + # Sync iterator def __iter__(self): return self diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index a569a27203..c7ed82e0d6 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1,16 +1,18 @@ -import json import os import sys from unittest.mock import MagicMock -import pytest -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path from litellm.llms.anthropic.chat.handler import ModelResponseIterator +from litellm.types.llms.openai import ( + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, +) +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME def test_redacted_thinking_content_block_delta(): @@ -39,3 +41,229 @@ def test_redacted_thinking_content_block_delta(): assert model_response.choices[0].delta.provider_specific_fields is not None assert "thinking_blocks" in model_response.choices[0].delta.provider_specific_fields + + +def test_handle_json_mode_chunk_response_format_tool(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + response_format_tool = ChatCompletionToolCallChunk( + id="tool_123", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=RESPONSE_FORMAT_TOOL_NAME, + arguments='{"question": "What is the weather?", "answer": "It is sunny"}', + ), + index=0, + ) + + text, tool_use = model_response_iterator._handle_json_mode_chunk( + "", response_format_tool + ) + print(f"\n\nresponse_format_tool text: {text}\n\n") + print(f"\n\nresponse_format_tool tool_use: {tool_use}\n\n") + + assert text == '{"question": "What is the weather?", "answer": "It is sunny"}' + assert tool_use is None + + +def test_handle_json_mode_chunk_regular_tool(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + regular_tool = ChatCompletionToolCallChunk( + id="tool_456", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name="get_weather", arguments='{"location": "San Francisco, CA"}' + ), + index=0, + ) + + text, tool_use = model_response_iterator._handle_json_mode_chunk("", regular_tool) + print(f"\n\nregular_tool text: {text}\n\n") + print(f"\n\nregular_tool tool_use: {tool_use}\n\n") + + assert text == "" + assert tool_use is not None + assert tool_use["function"]["name"] == "get_weather" + + +def test_handle_json_mode_chunk_streaming_response_format_tool(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + + # First chunk: response_format tool with id and name, but no arguments + first_chunk = ChatCompletionToolCallChunk( + id="tool_123", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=RESPONSE_FORMAT_TOOL_NAME, arguments="" + ), + index=0, + ) + + # Second chunk: continuation with arguments delta (no id) + second_chunk = ChatCompletionToolCallChunk( + id=None, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, arguments='{"question": "What is the weather?"' + ), + index=0, + ) + + # Third chunk: more arguments delta (no id) + third_chunk = ChatCompletionToolCallChunk( + id=None, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, arguments=', "answer": "It is sunny"}' + ), + index=0, + ) + + # Process first chunk - should set tracking flag but not convert yet (no args) + text1, tool_use1 = model_response_iterator._handle_json_mode_chunk("", first_chunk) + print(f"\n\nfirst_chunk text: {text1}\n\n") + print(f"\n\nfirst_chunk tool_use: {tool_use1}\n\n") + + # Process second chunk - should convert arguments to text + text2, tool_use2 = model_response_iterator._handle_json_mode_chunk("", second_chunk) + print(f"\n\nsecond_chunk text: {text2}\n\n") + print(f"\n\nsecond_chunk tool_use: {tool_use2}\n\n") + + # Process third chunk - should convert arguments to text + text3, tool_use3 = model_response_iterator._handle_json_mode_chunk("", third_chunk) + print(f"\n\nthird_chunk text: {text3}\n\n") + print(f"\n\nthird_chunk tool_use: {tool_use3}\n\n") + + # Verify response_format tool chunks are converted to content + assert text1 == "" # First chunk has no arguments + assert tool_use1 is None # Tool call suppressed + + assert text2 == '{"question": "What is the weather?"' # Second chunk arguments + assert tool_use2 is None # Tool call suppressed + + assert text3 == ', "answer": "It is sunny"}' # Third chunk arguments + assert tool_use3 is None # Tool call suppressed + + +def test_handle_json_mode_chunk_streaming_regular_tool(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + + # First chunk: regular tool with id and name, but no arguments + first_chunk = ChatCompletionToolCallChunk( + id="tool_456", + type="function", + function=ChatCompletionToolCallFunctionChunk(name="get_weather", arguments=""), + index=0, + ) + + # Second chunk: continuation with arguments delta (no id) + second_chunk = ChatCompletionToolCallChunk( + id=None, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, arguments='{"location": "San Francisco, CA"}' + ), + index=0, + ) + + # Process first chunk - should pass through as regular tool + text1, tool_use1 = model_response_iterator._handle_json_mode_chunk("", first_chunk) + print(f"\n\nregular first_chunk text: {text1}\n\n") + print(f"\n\nregular first_chunk tool_use: {tool_use1}\n\n") + + # Process second chunk - should pass through as regular tool + text2, tool_use2 = model_response_iterator._handle_json_mode_chunk("", second_chunk) + print(f"\n\nregular second_chunk text: {text2}\n\n") + print(f"\n\nregular second_chunk tool_use: {tool_use2}\n\n") + + # Verify regular tool chunks are passed through unchanged + assert text1 == "" # Original text unchanged + assert tool_use1 is not None # Tool call preserved + assert tool_use1["function"]["name"] == "get_weather" + + assert text2 == "" # Original text unchanged + assert tool_use2 is not None # Tool call preserved + assert tool_use2["function"]["arguments"] == '{"location": "San Francisco, CA"}' + + +def test_response_format_tool_finish_reason(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + + # First chunk: response_format tool + response_format_tool = ChatCompletionToolCallChunk( + id="tool_123", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=RESPONSE_FORMAT_TOOL_NAME, arguments='{"answer": "test"}' + ), + index=0, + ) + + # Process the tool call (should set converted_response_format_tool flag) + text, tool_use = model_response_iterator._handle_json_mode_chunk( + "", response_format_tool + ) + print( + f"\n\nconverted_response_format_tool flag: {model_response_iterator.converted_response_format_tool}\n\n" + ) + + # Simulate message_delta chunk with tool_use stop_reason + message_delta_chunk = { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 10}, + } + + # Process the message_delta chunk + model_response = model_response_iterator.chunk_parser(message_delta_chunk) + print(f"\n\nfinish_reason: {model_response.choices[0].finish_reason}\n\n") + + # Verify that finish_reason is overridden to "stop" for response_format tools + assert model_response_iterator.converted_response_format_tool is True + assert model_response.choices[0].finish_reason == "stop" + + +def test_regular_tool_finish_reason(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=True + ) + + # First chunk: regular tool (not response_format) + regular_tool = ChatCompletionToolCallChunk( + id="tool_456", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name="get_weather", arguments='{"location": "San Francisco, CA"}' + ), + index=0, + ) + + # Process the tool call (should NOT set converted_response_format_tool flag) + text, tool_use = model_response_iterator._handle_json_mode_chunk("", regular_tool) + print( + f"\n\nconverted_response_format_tool flag: {model_response_iterator.converted_response_format_tool}\n\n" + ) + + # Simulate message_delta chunk with tool_use stop_reason + message_delta_chunk = { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 10}, + } + + # Process the message_delta chunk + model_response = model_response_iterator.chunk_parser(message_delta_chunk) + print(f"\n\nfinish_reason: {model_response.choices[0].finish_reason}\n\n") + + # Verify that finish_reason remains "tool_calls" for regular tools + assert model_response_iterator.converted_response_format_tool is False + assert model_response.choices[0].finish_reason == "tool_calls"