diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index dee826e578..4afd4427d0 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2922,6 +2922,7 @@ class ModelResponseIterator: self.logging_obj = logging_obj self.is_function_call = check_is_function_call(logging_obj) self.cumulative_tool_call_index: int = 0 + self.has_seen_tool_calls: bool = False def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]: try: @@ -2960,6 +2961,40 @@ class ModelResponseIterator: cumulative_tool_call_index=self.cumulative_tool_call_index, ) + # Track whether tool_calls have been seen across streaming chunks. + # Gemini sends tool_calls and finishReason in separate chunks, + # so we need to remember if earlier chunks contained tool_calls + # to correctly set finish_reason="tool_calls" per the OpenAI spec. + if not self.has_seen_tool_calls: + for choice in model_response.choices: + if hasattr(choice, "delta") and choice.delta and choice.delta.tool_calls: + self.has_seen_tool_calls = True + break + + # Handle final chunk with finishReason but no content. + # _process_candidates skips candidates without "content", + # so the finish_reason from the final chunk is lost. + if not model_response.choices and _candidates: + from litellm.types.utils import Delta, StreamingChoices + + for candidate in _candidates: + finish_reason_str = candidate.get("finishReason") + if finish_reason_str is not None: + if self.has_seen_tool_calls: + mapped_finish_reason = "tool_calls" + else: + mapped_finish_reason = VertexGeminiConfig._check_finish_reason( + None, finish_reason_str + ) + choice = StreamingChoices( + finish_reason=mapped_finish_reason, + index=candidate.get("index", 0), + delta=Delta(content=None, role=None), + logprobs=None, + enhancements=None, + ) + model_response.choices.append(choice) + setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py b/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py new file mode 100644 index 0000000000..3f8efd47fa --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py @@ -0,0 +1,232 @@ +""" +Tests for Gemini streaming tool call finish_reason mapping. + +Gemini returns finishReason: "STOP" even when tool calls are present. +Per the OpenAI spec, finish_reason must be "tool_calls" when the model +called a tool. The ModelResponseIterator must track tool_calls across +streaming chunks and correctly set finish_reason on the final chunk. + +Ref: https://github.com/BerriAI/litellm/issues/21041 +""" + +from unittest.mock import MagicMock + +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + ModelResponseIterator, +) + + +def _make_logging_obj(**kwargs): + """Create a minimal mock logging object for ModelResponseIterator.""" + logging_obj = MagicMock() + logging_obj.optional_params = kwargs.get("optional_params", {}) + return logging_obj + + +def test_streaming_tool_call_finish_reason_is_tool_calls(): + """ + When Gemini streams tool calls across two chunks: + - Chunk 1: has tool call parts, no finishReason + - Chunk 2: has finishReason="STOP", no content + + The final chunk must have finish_reason="tool_calls" (not "stop"). + """ + logging_obj = _make_logging_obj() + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + logging_obj=logging_obj, + ) + + # Chunk 1: tool call with no finishReason + chunk_with_tool_calls = { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "Boston, MA"}, + } + } + ], + "role": "model", + }, + "index": 0, + } + ], + } + + # Chunk 2: finishReason="STOP" with no content + chunk_with_finish_reason = { + "candidates": [ + { + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 50, + "candidatesTokenCount": 20, + "totalTokenCount": 70, + }, + } + + # Process chunk 1 + response1 = iterator.chunk_parser(chunk_with_tool_calls) + assert response1 is not None + assert len(response1.choices) == 1 + assert response1.choices[0].delta.tool_calls is not None + assert response1.choices[0].finish_reason == "tool_calls" + assert iterator.has_seen_tool_calls is True + + # Process chunk 2 (final chunk) + response2 = iterator.chunk_parser(chunk_with_finish_reason) + assert response2 is not None + assert len(response2.choices) == 1 + assert response2.choices[0].finish_reason == "tool_calls" + + +def test_streaming_no_tool_calls_finish_reason_is_stop(): + """ + When Gemini streams a regular text response (no tool calls), + the final chunk with finishReason="STOP" should map to "stop". + """ + logging_obj = _make_logging_obj() + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + logging_obj=logging_obj, + ) + + # Chunk 1: text content, no finishReason + chunk_with_text = { + "candidates": [ + { + "content": { + "parts": [{"text": "Hello! How can I help?"}], + "role": "model", + }, + "index": 0, + } + ], + } + + # Chunk 2: finishReason="STOP" with no content + chunk_with_finish_reason = { + "candidates": [ + { + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 8, + "totalTokenCount": 18, + }, + } + + # Process chunk 1 + response1 = iterator.chunk_parser(chunk_with_text) + assert response1 is not None + assert len(response1.choices) == 1 + assert iterator.has_seen_tool_calls is False + + # Process chunk 2 + response2 = iterator.chunk_parser(chunk_with_finish_reason) + assert response2 is not None + assert len(response2.choices) == 1 + assert response2.choices[0].finish_reason == "stop" + + +def test_streaming_multiple_tool_calls_finish_reason(): + """ + When Gemini streams multiple tool calls across chunks, + the final finish_reason must still be "tool_calls". + """ + logging_obj = _make_logging_obj() + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + logging_obj=logging_obj, + ) + + # Chunk 1: first tool call + chunk_tool_1 = { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "NYC"}, + } + }, + { + "functionCall": { + "name": "get_time", + "args": {"timezone": "EST"}, + } + }, + ], + "role": "model", + }, + "index": 0, + } + ], + } + + # Chunk 2: finishReason="STOP" with no content + chunk_finish = { + "candidates": [ + { + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 50, + "candidatesTokenCount": 30, + "totalTokenCount": 80, + }, + } + + response1 = iterator.chunk_parser(chunk_tool_1) + assert response1 is not None + assert iterator.has_seen_tool_calls is True + + response2 = iterator.chunk_parser(chunk_finish) + assert response2 is not None + assert len(response2.choices) == 1 + assert response2.choices[0].finish_reason == "tool_calls" + + +def test_streaming_content_filter_finish_reason_preserved(): + """ + When Gemini returns finishReason due to content filtering (not STOP), + and no tool calls were seen, the content_filter reason should be preserved. + """ + logging_obj = _make_logging_obj() + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + logging_obj=logging_obj, + ) + + # Chunk with finishReason="SAFETY" and no content + chunk_safety = { + "candidates": [ + { + "finishReason": "SAFETY", + "index": 0, + } + ], + } + + response = iterator.chunk_parser(chunk_safety) + assert response is not None + assert len(response.choices) == 1 + assert response.choices[0].finish_reason == "content_filter"