mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-23 04:28:07 +00:00
fix(anthropic): fix streaming + response_format + tools bug (#12463)
* fix(anthropic): fix streaming + response_format + tools bug - Fix _handle_json_mode_chunk to only convert response_format tools to content - Regular user tools now remain as proper tool_calls in streaming mode - Add comprehensive test for the fix - Resolves issue where all tools were incorrectly converted to content chunks Before: All tools converted to content with different indices After: Only response_format tool converted, regular tools remain as tool_calls * fix(anthropic): improve streaming + response_format + tools handling * fix: lint error (too many statements) * fix(anthropic): correct finish_reason for streaming response_format tools
This commit is contained in:
Dan McAulay
@@ -22,6 +22,7 @@ import litellm
|
||||
import litellm.litellm_core_utils
|
||||
import litellm.types
|
||||
import litellm.types.utils
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
@@ -487,6 +488,11 @@ class ModelResponseIterator:
|
||||
self.tool_index = -1
|
||||
self.json_mode = json_mode
|
||||
|
||||
# Track if we're currently streaming a response_format tool
|
||||
self.is_response_format_tool: bool = False
|
||||
# Track if we've converted any response_format tools (affects finish_reason)
|
||||
self.converted_response_format_tool: bool = False
|
||||
|
||||
def check_empty_tool_call_args(self) -> bool:
|
||||
"""
|
||||
Check if the tool call block so far has been an empty string
|
||||
@@ -515,7 +521,9 @@ class ModelResponseIterator:
|
||||
usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None
|
||||
)
|
||||
|
||||
def _content_block_delta_helper(self, chunk: dict) -> Tuple[
|
||||
def _content_block_delta_helper(
|
||||
self, chunk: dict
|
||||
) -> Tuple[
|
||||
str,
|
||||
Optional[ChatCompletionToolCallChunk],
|
||||
List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]],
|
||||
@@ -680,7 +688,6 @@ class ModelResponseIterator:
|
||||
ContentBlockStop(**chunk) # type: ignore
|
||||
# check if tool call content block
|
||||
is_empty = self.check_empty_tool_call_args()
|
||||
|
||||
if is_empty:
|
||||
tool_use = {
|
||||
"id": None,
|
||||
@@ -691,18 +698,10 @@ class ModelResponseIterator:
|
||||
},
|
||||
"index": self.tool_index,
|
||||
}
|
||||
# Reset response_format tool tracking when block stops
|
||||
self.is_response_format_tool = False
|
||||
elif type_chunk == "message_delta":
|
||||
"""
|
||||
Anthropic
|
||||
chunk = {'type': 'message_delta', 'delta': {'stop_reason': 'max_tokens', 'stop_sequence': None}, 'usage': {'output_tokens': 10}}
|
||||
"""
|
||||
# TODO - get usage from this chunk, set in response
|
||||
message_delta = MessageBlockDelta(**chunk) # type: ignore
|
||||
finish_reason = map_finish_reason(
|
||||
finish_reason=message_delta["delta"].get("stop_reason", "stop")
|
||||
or "stop"
|
||||
)
|
||||
usage = self._handle_usage(anthropic_usage_chunk=message_delta["usage"])
|
||||
finish_reason, usage = self._handle_message_delta(chunk)
|
||||
elif type_chunk == "message_start":
|
||||
"""
|
||||
Anthropic
|
||||
@@ -778,6 +777,13 @@ class ModelResponseIterator:
|
||||
Anthropic returns the JSON schema as part of the tool call
|
||||
OpenAI returns the JSON schema as part of the content, this handles placing it in the content
|
||||
|
||||
Tool streaming follows Anthropic's fine-grained streaming pattern:
|
||||
- content_block_start: Contains complete tool info (id, name, empty arguments)
|
||||
- content_block_delta: Contains argument deltas (partial_json)
|
||||
- content_block_stop: Signals end of tool
|
||||
|
||||
Reference: https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/fine-grained-tool-streaming
|
||||
|
||||
Args:
|
||||
text: str
|
||||
tool_use: Optional[ChatCompletionToolCallChunk]
|
||||
@@ -787,16 +793,50 @@ class ModelResponseIterator:
|
||||
text: The text to use in the content
|
||||
tool_use: The ChatCompletionToolCallChunk to use in the chunk response
|
||||
"""
|
||||
if self.json_mode is True and tool_use is not None:
|
||||
if not self.json_mode or tool_use is None:
|
||||
return text, tool_use
|
||||
|
||||
# Check if this is a new tool call (has id)
|
||||
if tool_use.get("id") is not None:
|
||||
# New tool call from content_block_start - tool name is always complete here
|
||||
# (per Anthropic's fine-grained streaming pattern)
|
||||
tool_name = tool_use.get("function", {}).get("name", "")
|
||||
self.is_response_format_tool = tool_name == RESPONSE_FORMAT_TOOL_NAME
|
||||
|
||||
# Convert tool to content if we're tracking a response_format tool
|
||||
if self.is_response_format_tool:
|
||||
message = AnthropicConfig._convert_tool_response_to_message(
|
||||
tool_calls=[tool_use]
|
||||
)
|
||||
if message is not None:
|
||||
text = message.content or ""
|
||||
tool_use = None
|
||||
# Track that we converted a response_format tool
|
||||
self.converted_response_format_tool = True
|
||||
|
||||
return text, tool_use
|
||||
|
||||
def _handle_message_delta(self, chunk: dict) -> Tuple[str, Optional[Usage]]:
|
||||
"""
|
||||
Handle message_delta event for finish_reason and usage.
|
||||
|
||||
Args:
|
||||
chunk: The message_delta chunk
|
||||
|
||||
Returns:
|
||||
Tuple of (finish_reason, usage)
|
||||
"""
|
||||
message_delta = MessageBlockDelta(**chunk) # type: ignore
|
||||
finish_reason = map_finish_reason(
|
||||
finish_reason=message_delta["delta"].get("stop_reason", "stop") or "stop"
|
||||
)
|
||||
# Override finish_reason to "stop" if we converted response_format tools
|
||||
# (matches OpenAI behavior and non-streaming Anthropic implementation)
|
||||
if self.converted_response_format_tool:
|
||||
finish_reason = "stop"
|
||||
usage = self._handle_usage(anthropic_usage_chunk=message_delta["usage"])
|
||||
return finish_reason, usage
|
||||
|
||||
# Sync iterator
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
)
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
|
||||
|
||||
def test_redacted_thinking_content_block_delta():
|
||||
@@ -39,3 +41,229 @@ def test_redacted_thinking_content_block_delta():
|
||||
|
||||
assert model_response.choices[0].delta.provider_specific_fields is not None
|
||||
assert "thinking_blocks" in model_response.choices[0].delta.provider_specific_fields
|
||||
|
||||
|
||||
def test_handle_json_mode_chunk_response_format_tool():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
response_format_tool = ChatCompletionToolCallChunk(
|
||||
id="tool_123",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=RESPONSE_FORMAT_TOOL_NAME,
|
||||
arguments='{"question": "What is the weather?", "answer": "It is sunny"}',
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
text, tool_use = model_response_iterator._handle_json_mode_chunk(
|
||||
"", response_format_tool
|
||||
)
|
||||
print(f"\n\nresponse_format_tool text: {text}\n\n")
|
||||
print(f"\n\nresponse_format_tool tool_use: {tool_use}\n\n")
|
||||
|
||||
assert text == '{"question": "What is the weather?", "answer": "It is sunny"}'
|
||||
assert tool_use is None
|
||||
|
||||
|
||||
def test_handle_json_mode_chunk_regular_tool():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
regular_tool = ChatCompletionToolCallChunk(
|
||||
id="tool_456",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name="get_weather", arguments='{"location": "San Francisco, CA"}'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
text, tool_use = model_response_iterator._handle_json_mode_chunk("", regular_tool)
|
||||
print(f"\n\nregular_tool text: {text}\n\n")
|
||||
print(f"\n\nregular_tool tool_use: {tool_use}\n\n")
|
||||
|
||||
assert text == ""
|
||||
assert tool_use is not None
|
||||
assert tool_use["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_handle_json_mode_chunk_streaming_response_format_tool():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
|
||||
# First chunk: response_format tool with id and name, but no arguments
|
||||
first_chunk = ChatCompletionToolCallChunk(
|
||||
id="tool_123",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=RESPONSE_FORMAT_TOOL_NAME, arguments=""
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Second chunk: continuation with arguments delta (no id)
|
||||
second_chunk = ChatCompletionToolCallChunk(
|
||||
id=None,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=None, arguments='{"question": "What is the weather?"'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Third chunk: more arguments delta (no id)
|
||||
third_chunk = ChatCompletionToolCallChunk(
|
||||
id=None,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=None, arguments=', "answer": "It is sunny"}'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Process first chunk - should set tracking flag but not convert yet (no args)
|
||||
text1, tool_use1 = model_response_iterator._handle_json_mode_chunk("", first_chunk)
|
||||
print(f"\n\nfirst_chunk text: {text1}\n\n")
|
||||
print(f"\n\nfirst_chunk tool_use: {tool_use1}\n\n")
|
||||
|
||||
# Process second chunk - should convert arguments to text
|
||||
text2, tool_use2 = model_response_iterator._handle_json_mode_chunk("", second_chunk)
|
||||
print(f"\n\nsecond_chunk text: {text2}\n\n")
|
||||
print(f"\n\nsecond_chunk tool_use: {tool_use2}\n\n")
|
||||
|
||||
# Process third chunk - should convert arguments to text
|
||||
text3, tool_use3 = model_response_iterator._handle_json_mode_chunk("", third_chunk)
|
||||
print(f"\n\nthird_chunk text: {text3}\n\n")
|
||||
print(f"\n\nthird_chunk tool_use: {tool_use3}\n\n")
|
||||
|
||||
# Verify response_format tool chunks are converted to content
|
||||
assert text1 == "" # First chunk has no arguments
|
||||
assert tool_use1 is None # Tool call suppressed
|
||||
|
||||
assert text2 == '{"question": "What is the weather?"' # Second chunk arguments
|
||||
assert tool_use2 is None # Tool call suppressed
|
||||
|
||||
assert text3 == ', "answer": "It is sunny"}' # Third chunk arguments
|
||||
assert tool_use3 is None # Tool call suppressed
|
||||
|
||||
|
||||
def test_handle_json_mode_chunk_streaming_regular_tool():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
|
||||
# First chunk: regular tool with id and name, but no arguments
|
||||
first_chunk = ChatCompletionToolCallChunk(
|
||||
id="tool_456",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(name="get_weather", arguments=""),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Second chunk: continuation with arguments delta (no id)
|
||||
second_chunk = ChatCompletionToolCallChunk(
|
||||
id=None,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=None, arguments='{"location": "San Francisco, CA"}'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Process first chunk - should pass through as regular tool
|
||||
text1, tool_use1 = model_response_iterator._handle_json_mode_chunk("", first_chunk)
|
||||
print(f"\n\nregular first_chunk text: {text1}\n\n")
|
||||
print(f"\n\nregular first_chunk tool_use: {tool_use1}\n\n")
|
||||
|
||||
# Process second chunk - should pass through as regular tool
|
||||
text2, tool_use2 = model_response_iterator._handle_json_mode_chunk("", second_chunk)
|
||||
print(f"\n\nregular second_chunk text: {text2}\n\n")
|
||||
print(f"\n\nregular second_chunk tool_use: {tool_use2}\n\n")
|
||||
|
||||
# Verify regular tool chunks are passed through unchanged
|
||||
assert text1 == "" # Original text unchanged
|
||||
assert tool_use1 is not None # Tool call preserved
|
||||
assert tool_use1["function"]["name"] == "get_weather"
|
||||
|
||||
assert text2 == "" # Original text unchanged
|
||||
assert tool_use2 is not None # Tool call preserved
|
||||
assert tool_use2["function"]["arguments"] == '{"location": "San Francisco, CA"}'
|
||||
|
||||
|
||||
def test_response_format_tool_finish_reason():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
|
||||
# First chunk: response_format tool
|
||||
response_format_tool = ChatCompletionToolCallChunk(
|
||||
id="tool_123",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=RESPONSE_FORMAT_TOOL_NAME, arguments='{"answer": "test"}'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Process the tool call (should set converted_response_format_tool flag)
|
||||
text, tool_use = model_response_iterator._handle_json_mode_chunk(
|
||||
"", response_format_tool
|
||||
)
|
||||
print(
|
||||
f"\n\nconverted_response_format_tool flag: {model_response_iterator.converted_response_format_tool}\n\n"
|
||||
)
|
||||
|
||||
# Simulate message_delta chunk with tool_use stop_reason
|
||||
message_delta_chunk = {
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 10},
|
||||
}
|
||||
|
||||
# Process the message_delta chunk
|
||||
model_response = model_response_iterator.chunk_parser(message_delta_chunk)
|
||||
print(f"\n\nfinish_reason: {model_response.choices[0].finish_reason}\n\n")
|
||||
|
||||
# Verify that finish_reason is overridden to "stop" for response_format tools
|
||||
assert model_response_iterator.converted_response_format_tool is True
|
||||
assert model_response.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def test_regular_tool_finish_reason():
|
||||
model_response_iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=True
|
||||
)
|
||||
|
||||
# First chunk: regular tool (not response_format)
|
||||
regular_tool = ChatCompletionToolCallChunk(
|
||||
id="tool_456",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name="get_weather", arguments='{"location": "San Francisco, CA"}'
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
|
||||
# Process the tool call (should NOT set converted_response_format_tool flag)
|
||||
text, tool_use = model_response_iterator._handle_json_mode_chunk("", regular_tool)
|
||||
print(
|
||||
f"\n\nconverted_response_format_tool flag: {model_response_iterator.converted_response_format_tool}\n\n"
|
||||
)
|
||||
|
||||
# Simulate message_delta chunk with tool_use stop_reason
|
||||
message_delta_chunk = {
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 10},
|
||||
}
|
||||
|
||||
# Process the message_delta chunk
|
||||
model_response = model_response_iterator.chunk_parser(message_delta_chunk)
|
||||
print(f"\n\nfinish_reason: {model_response.choices[0].finish_reason}\n\n")
|
||||
|
||||
# Verify that finish_reason remains "tool_calls" for regular tools
|
||||
assert model_response_iterator.converted_response_format_tool is False
|
||||
assert model_response.choices[0].finish_reason == "tool_calls"
|
||||
|
||||
Reference in New Issue
Block a user