tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-14 08:26:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20712 from emerzon/fix/responses-tool-call-delta-index-mapping

Browse Source 
fix(responses): preserve tool call argument deltas when streaming id is omitted
This commit is contained in:
Sameer Kankute
2026-02-09 08:43:04 +05:30
committed by GitHub
parent 4f96a3b126 cf17a440cd
commit aaa48f8ad2
2 changed files with 196 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/responses/litellm_completion_transformation/streaming_iterator.py
+35 -2
View File
@@ -88,6 +88,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._pending_tool_events: List[BaseLiteLLMOpenAIResponseObject] = []
self._tool_output_index_by_call_id: dict[str, int] = {}
self._tool_args_by_call_id: dict[str, str] = {}
self._tool_call_id_by_index: dict[int, str] = {}
self._ambiguous_tool_call_indexes: set[int] = set()
self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item
self._final_tool_events_queued: bool = False
self._sequence_number: int = 0
@@ -111,6 +113,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._tool_output_index_by_call_id[call_id] = idx
return idx
def _normalize_tool_call_index(self, tool_call: object) -> Optional[int]:
idx_raw = (
tool_call.get("index")
if isinstance(tool_call, dict)
else getattr(tool_call, "index", None)
)
if idx_raw is None:
return None
try:
return int(idx_raw)
except (TypeError, ValueError):
return None
def _is_reasoning_end(self, chunk):
delta = chunk.choices[0].delta
@@ -143,10 +158,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return
for tc in tool_calls:
tc_index = self._normalize_tool_call_index(tc)
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not call_id_raw:
call_id = ""
if call_id_raw:
call_id = str(call_id_raw)
if tc_index is not None:
existing_call_id = self._tool_call_id_by_index.get(tc_index)
if existing_call_id is not None and existing_call_id != call_id:
# Reusing the same index for multiple call_ids is ambiguous for id-less deltas.
# Guard against silent misrouting by disabling index fallback for this index.
self._ambiguous_tool_call_indexes.add(tc_index)
self._tool_call_id_by_index[tc_index] = call_id
elif tc_index is not None:
if tc_index in self._ambiguous_tool_call_indexes:
continue
mapped_call_id = self._tool_call_id_by_index.get(tc_index)
if mapped_call_id:
call_id = mapped_call_id
if not call_id:
continue
call_id = str(call_id_raw)
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
fn_name = ""
tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py
+161
View File
@@ -229,3 +229,164 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
assert sequence_numbers == sorted(sequence_numbers)
assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique
def test_tool_call_delta_without_id_uses_index_mapping():
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=AsyncMock(),
request_input="Test input",
responses_api_request={},
)
chunks = [
[
{
"index": 0,
"id": "call_abc123",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"lo'},
}
],
[{"index": 0, "type": "function", "function": {"arguments": 'cation":'}}],
[{"index": 0, "type": "function", "function": {"arguments": ' "New'}}],
[{"index": 0, "type": "function", "function": {"arguments": ' York"}'}}],
]
for tool_calls in chunks:
iterator._queue_tool_call_delta_events(tool_calls)
all_events = []
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
streamed_arguments = "".join(evt.delta for evt in delta_events)
assert streamed_arguments == '{"location": "New York"}'
output_item_added_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
]
assert len(output_item_added_events) == 1
assert output_item_added_events[0].item.id == "call_abc123"
def test_parallel_tool_calls_without_ids_use_index_mapping():
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=AsyncMock(),
request_input="Test input",
responses_api_request={},
)
iterator._queue_tool_call_delta_events(
[
{
"index": 0,
"id": "call_a",
"type": "function",
"function": {"name": "tool_a", "arguments": '{"x":'},
},
{
"index": 1,
"id": "call_b",
"type": "function",
"function": {"name": "tool_b", "arguments": '{"y":'},
},
]
)
iterator._queue_tool_call_delta_events(
[
{"index": 0, "type": "function", "function": {"arguments": "1}"}},
{"index": 1, "type": "function", "function": {"arguments": "2}"}},
]
)
all_events = []
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
output_item_added_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
]
assert len(output_item_added_events) == 2
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
arguments_by_call_id = {}
for evt in delta_events:
arguments_by_call_id.setdefault(evt.item_id, "")
arguments_by_call_id[evt.item_id] += evt.delta
assert arguments_by_call_id["call_a"] == '{"x":1}'
assert arguments_by_call_id["call_b"] == '{"y":2}'
def test_reused_index_with_new_call_id_marks_fallback_ambiguous():
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=AsyncMock(),
request_input="Test input",
responses_api_request={},
)
iterator._queue_tool_call_delta_events(
[
{
"index": 0,
"id": "call_a",
"type": "function",
"function": {"name": "tool_a", "arguments": '{"a":'},
}
]
)
iterator._queue_tool_call_delta_events(
[
{
"index": 0,
"id": "call_b",
"type": "function",
"function": {"name": "tool_b", "arguments": '{"b":'},
}
]
)
# Ambiguous chunk: index reused and id missing. We should skip fallback rather than misroute.
iterator._queue_tool_call_delta_events(
[
{
"index": 0,
"type": "function",
"function": {"arguments": "1}"},
}
]
)
all_events = []
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
arguments_by_call_id = {}
for evt in delta_events:
arguments_by_call_id.setdefault(evt.item_id, "")
arguments_by_call_id[evt.item_id] += evt.delta
assert arguments_by_call_id["call_a"] == '{"a":'
assert arguments_by_call_id["call_b"] == '{"b":'
assert arguments_by_call_id["call_a"] != '{"a":1}'
assert arguments_by_call_id["call_b"] != '{"b":1}'