diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 867c18b6dd..5c05526442 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -88,6 +88,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._pending_tool_events: List[BaseLiteLLMOpenAIResponseObject] = [] self._tool_output_index_by_call_id: dict[str, int] = {} self._tool_args_by_call_id: dict[str, str] = {} + self._tool_call_id_by_index: dict[int, str] = {} + self._ambiguous_tool_call_indexes: set[int] = set() self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item self._final_tool_events_queued: bool = False self._sequence_number: int = 0 @@ -111,6 +113,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_output_index_by_call_id[call_id] = idx return idx + def _normalize_tool_call_index(self, tool_call: object) -> Optional[int]: + idx_raw = ( + tool_call.get("index") + if isinstance(tool_call, dict) + else getattr(tool_call, "index", None) + ) + if idx_raw is None: + return None + try: + return int(idx_raw) + except (TypeError, ValueError): + return None + def _is_reasoning_end(self, chunk): delta = chunk.choices[0].delta @@ -143,10 +158,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return for tc in tool_calls: + tc_index = self._normalize_tool_call_index(tc) call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) - if not call_id_raw: + call_id = "" + + if call_id_raw: + call_id = str(call_id_raw) + if tc_index is not None: + existing_call_id = self._tool_call_id_by_index.get(tc_index) + if existing_call_id is not None and existing_call_id != call_id: + # Reusing the same index for multiple call_ids is ambiguous for id-less deltas. + # Guard against silent misrouting by disabling index fallback for this index. + self._ambiguous_tool_call_indexes.add(tc_index) + self._tool_call_id_by_index[tc_index] = call_id + elif tc_index is not None: + if tc_index in self._ambiguous_tool_call_indexes: + continue + mapped_call_id = self._tool_call_id_by_index.get(tc_index) + if mapped_call_id: + call_id = mapped_call_id + + if not call_id: continue - call_id = str(call_id_raw) fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) fn_name = "" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py index 8d324bea61..071eefaef4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py @@ -229,3 +229,164 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): assert sequence_numbers == sorted(sequence_numbers) assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique + +def test_tool_call_delta_without_id_uses_index_mapping(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + chunks = [ + [ + { + "index": 0, + "id": "call_abc123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"lo'}, + } + ], + [{"index": 0, "type": "function", "function": {"arguments": 'cation":'}}], + [{"index": 0, "type": "function", "function": {"arguments": ' "New'}}], + [{"index": 0, "type": "function", "function": {"arguments": ' York"}'}}], + ] + + for tool_calls in chunks: + iterator._queue_tool_call_delta_events(tool_calls) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + streamed_arguments = "".join(evt.delta for evt in delta_events) + + assert streamed_arguments == '{"location": "New York"}' + + output_item_added_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert len(output_item_added_events) == 1 + assert output_item_added_events[0].item.id == "call_abc123" + + +def test_parallel_tool_calls_without_ids_use_index_mapping(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_a", + "type": "function", + "function": {"name": "tool_a", "arguments": '{"x":'}, + }, + { + "index": 1, + "id": "call_b", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"y":'}, + }, + ] + ) + iterator._queue_tool_call_delta_events( + [ + {"index": 0, "type": "function", "function": {"arguments": "1}"}}, + {"index": 1, "type": "function", "function": {"arguments": "2}"}}, + ] + ) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + output_item_added_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert len(output_item_added_events) == 2 + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + arguments_by_call_id = {} + for evt in delta_events: + arguments_by_call_id.setdefault(evt.item_id, "") + arguments_by_call_id[evt.item_id] += evt.delta + + assert arguments_by_call_id["call_a"] == '{"x":1}' + assert arguments_by_call_id["call_b"] == '{"y":2}' + + +def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_a", + "type": "function", + "function": {"name": "tool_a", "arguments": '{"a":'}, + } + ] + ) + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_b", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"b":'}, + } + ] + ) + # Ambiguous chunk: index reused and id missing. We should skip fallback rather than misroute. + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "type": "function", + "function": {"arguments": "1}"}, + } + ] + ) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + arguments_by_call_id = {} + for evt in delta_events: + arguments_by_call_id.setdefault(evt.item_id, "") + arguments_by_call_id[evt.item_id] += evt.delta + + assert arguments_by_call_id["call_a"] == '{"a":' + assert arguments_by_call_id["call_b"] == '{"b":' + assert arguments_by_call_id["call_a"] != '{"a":1}' + assert arguments_by_call_id["call_b"] != '{"b":1}'