Merge pull request #21577 from Chesars/fix/gemini-streaming-tool-calls-finish-reason

fix(gemini): correct streaming finish_reason for tool calls

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -2922,6 +2922,7 @@ class ModelResponseIterator:
         self.logging_obj = logging_obj
         self.is_function_call = check_is_function_call(logging_obj)
         self.cumulative_tool_call_index: int = 0
+        self.has_seen_tool_calls: bool = False
 
     def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]:
         try:
@@ -2960,6 +2961,40 @@ class ModelResponseIterator:
                     cumulative_tool_call_index=self.cumulative_tool_call_index,
                 )
 
+                # Track whether tool_calls have been seen across streaming chunks.
+                # Gemini sends tool_calls and finishReason in separate chunks,
+                # so we need to remember if earlier chunks contained tool_calls
+                # to correctly set finish_reason="tool_calls" per the OpenAI spec.
+                if not self.has_seen_tool_calls:
+                    for choice in model_response.choices:
+                        if hasattr(choice, "delta") and choice.delta and choice.delta.tool_calls:
+                            self.has_seen_tool_calls = True
+                            break
+
+                # Handle final chunk with finishReason but no content.
+                # _process_candidates skips candidates without "content",
+                # so the finish_reason from the final chunk is lost.
+                if not model_response.choices and _candidates:
+                    from litellm.types.utils import Delta, StreamingChoices
+
+                    for candidate in _candidates:
+                        finish_reason_str = candidate.get("finishReason")
+                        if finish_reason_str is not None:
+                            if self.has_seen_tool_calls:
+                                mapped_finish_reason = "tool_calls"
+                            else:
+                                mapped_finish_reason = VertexGeminiConfig._check_finish_reason(
+                                    None, finish_reason_str
+                                )
+                            choice = StreamingChoices(
+                                finish_reason=mapped_finish_reason,
+                                index=candidate.get("index", 0),
+                                delta=Delta(content=None, role=None),
+                                logprobs=None,
+                                enhancements=None,
+                            )
+                            model_response.choices.append(choice)
+
                 setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)  # type: ignore
                 setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata)  # type: ignore
                 setattr(model_response, "vertex_ai_safety_ratings", safety_ratings)  # type: ignore

tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py

@@ -0,0 +1,232 @@
+"""
+Tests for Gemini streaming tool call finish_reason mapping.
+
+Gemini returns finishReason: "STOP" even when tool calls are present.
+Per the OpenAI spec, finish_reason must be "tool_calls" when the model
+called a tool. The ModelResponseIterator must track tool_calls across
+streaming chunks and correctly set finish_reason on the final chunk.
+
+Ref: https://github.com/BerriAI/litellm/issues/21041
+"""
+
+from unittest.mock import MagicMock
+
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+    ModelResponseIterator,
+)
+
+
+def _make_logging_obj(**kwargs):
+    """Create a minimal mock logging object for ModelResponseIterator."""
+    logging_obj = MagicMock()
+    logging_obj.optional_params = kwargs.get("optional_params", {})
+    return logging_obj
+
+
+def test_streaming_tool_call_finish_reason_is_tool_calls():
+    """
+    When Gemini streams tool calls across two chunks:
+      - Chunk 1: has tool call parts, no finishReason
+      - Chunk 2: has finishReason="STOP", no content
+
+    The final chunk must have finish_reason="tool_calls" (not "stop").
+    """
+    logging_obj = _make_logging_obj()
+    iterator = ModelResponseIterator(
+        streaming_response=iter([]),
+        sync_stream=True,
+        logging_obj=logging_obj,
+    )
+
+    # Chunk 1: tool call with no finishReason
+    chunk_with_tool_calls = {
+        "candidates": [
+            {
+                "content": {
+                    "parts": [
+                        {
+                            "functionCall": {
+                                "name": "get_current_weather",
+                                "args": {"location": "Boston, MA"},
+                            }
+                        }
+                    ],
+                    "role": "model",
+                },
+                "index": 0,
+            }
+        ],
+    }
+
+    # Chunk 2: finishReason="STOP" with no content
+    chunk_with_finish_reason = {
+        "candidates": [
+            {
+                "finishReason": "STOP",
+                "index": 0,
+            }
+        ],
+        "usageMetadata": {
+            "promptTokenCount": 50,
+            "candidatesTokenCount": 20,
+            "totalTokenCount": 70,
+        },
+    }
+
+    # Process chunk 1
+    response1 = iterator.chunk_parser(chunk_with_tool_calls)
+    assert response1 is not None
+    assert len(response1.choices) == 1
+    assert response1.choices[0].delta.tool_calls is not None
+    assert response1.choices[0].finish_reason == "tool_calls"
+    assert iterator.has_seen_tool_calls is True
+
+    # Process chunk 2 (final chunk)
+    response2 = iterator.chunk_parser(chunk_with_finish_reason)
+    assert response2 is not None
+    assert len(response2.choices) == 1
+    assert response2.choices[0].finish_reason == "tool_calls"
+
+
+def test_streaming_no_tool_calls_finish_reason_is_stop():
+    """
+    When Gemini streams a regular text response (no tool calls),
+    the final chunk with finishReason="STOP" should map to "stop".
+    """
+    logging_obj = _make_logging_obj()
+    iterator = ModelResponseIterator(
+        streaming_response=iter([]),
+        sync_stream=True,
+        logging_obj=logging_obj,
+    )
+
+    # Chunk 1: text content, no finishReason
+    chunk_with_text = {
+        "candidates": [
+            {
+                "content": {
+                    "parts": [{"text": "Hello! How can I help?"}],
+                    "role": "model",
+                },
+                "index": 0,
+            }
+        ],
+    }
+
+    # Chunk 2: finishReason="STOP" with no content
+    chunk_with_finish_reason = {
+        "candidates": [
+            {
+                "finishReason": "STOP",
+                "index": 0,
+            }
+        ],
+        "usageMetadata": {
+            "promptTokenCount": 10,
+            "candidatesTokenCount": 8,
+            "totalTokenCount": 18,
+        },
+    }
+
+    # Process chunk 1
+    response1 = iterator.chunk_parser(chunk_with_text)
+    assert response1 is not None
+    assert len(response1.choices) == 1
+    assert iterator.has_seen_tool_calls is False
+
+    # Process chunk 2
+    response2 = iterator.chunk_parser(chunk_with_finish_reason)
+    assert response2 is not None
+    assert len(response2.choices) == 1
+    assert response2.choices[0].finish_reason == "stop"
+
+
+def test_streaming_multiple_tool_calls_finish_reason():
+    """
+    When Gemini streams multiple tool calls across chunks,
+    the final finish_reason must still be "tool_calls".
+    """
+    logging_obj = _make_logging_obj()
+    iterator = ModelResponseIterator(
+        streaming_response=iter([]),
+        sync_stream=True,
+        logging_obj=logging_obj,
+    )
+
+    # Chunk 1: first tool call
+    chunk_tool_1 = {
+        "candidates": [
+            {
+                "content": {
+                    "parts": [
+                        {
+                            "functionCall": {
+                                "name": "get_weather",
+                                "args": {"location": "NYC"},
+                            }
+                        },
+                        {
+                            "functionCall": {
+                                "name": "get_time",
+                                "args": {"timezone": "EST"},
+                            }
+                        },
+                    ],
+                    "role": "model",
+                },
+                "index": 0,
+            }
+        ],
+    }
+
+    # Chunk 2: finishReason="STOP" with no content
+    chunk_finish = {
+        "candidates": [
+            {
+                "finishReason": "STOP",
+                "index": 0,
+            }
+        ],
+        "usageMetadata": {
+            "promptTokenCount": 50,
+            "candidatesTokenCount": 30,
+            "totalTokenCount": 80,
+        },
+    }
+
+    response1 = iterator.chunk_parser(chunk_tool_1)
+    assert response1 is not None
+    assert iterator.has_seen_tool_calls is True
+
+    response2 = iterator.chunk_parser(chunk_finish)
+    assert response2 is not None
+    assert len(response2.choices) == 1
+    assert response2.choices[0].finish_reason == "tool_calls"
+
+
+def test_streaming_content_filter_finish_reason_preserved():
+    """
+    When Gemini returns finishReason due to content filtering (not STOP),
+    and no tool calls were seen, the content_filter reason should be preserved.
+    """
+    logging_obj = _make_logging_obj()
+    iterator = ModelResponseIterator(
+        streaming_response=iter([]),
+        sync_stream=True,
+        logging_obj=logging_obj,
+    )
+
+    # Chunk with finishReason="SAFETY" and no content
+    chunk_safety = {
+        "candidates": [
+            {
+                "finishReason": "SAFETY",
+                "index": 0,
+            }
+        ],
+    }
+
+    response = iterator.chunk_parser(chunk_safety)
+    assert response is not None
+    assert len(response.choices) == 1
+    assert response.choices[0].finish_reason == "content_filter"