From 75c72c51e28eaeb5de977bfd4f33e1230d862fc9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 21 May 2026 03:22:50 +0530 Subject: [PATCH] fix(vertex_ai): omit function_call id on Vertex Gemini 3.5+ tool turns (#28324) * fix(vertex_ai): omit function_call id on Vertex Gemini 3.5+ tool turns Vertex AI rejects `id` on function_call/function_response parts; only Google AI Studio accepts it for Gemini 3.5+ strict tool matching. Co-authored-by: Cursor * Update litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(vertex_ai): forward custom_llm_provider in context caching Pass custom_llm_provider through to _gemini_convert_messages_with_history in the context caching path so Gemini 3.5+ tool-call `id` forwarding behaves consistently between cached and non-cached completions on Google AI Studio. Co-authored-by: Claude --------- Co-authored-by: Cursor Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> Co-authored-by: Claude Co-authored-by: Claude (cherry picked from commit fecf212d7001cd409139249a69e6cbaa1f00afd6) --- .../prompt_templates/factory.py | 15 +- litellm/llms/gemini/chat/transformation.py | 6 +- .../context_caching/transformation.py | 4 +- .../llms/vertex_ai/gemini/transformation.py | 6 +- .../vertex_and_google_ai_studio_gemini.py | 20 ++- litellm/types/llms/vertex_ai.py | 10 +- ...test_vertex_and_google_ai_studio_gemini.py | 148 +++++++++++++++++- 7 files changed, 188 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d39b7a5056..0663d4167d 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1344,6 +1344,7 @@ def _get_dummy_thought_signature() -> str: def convert_to_gemini_tool_call_invoke( message: ChatCompletionAssistantMessage, model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> List[VertexPartType]: """ OpenAI tool invokes: @@ -1394,7 +1395,10 @@ def convert_to_gemini_tool_call_invoke( ) forward_tool_call_id = bool( - model and VertexGeminiConfig._is_gemini_3_or_newer(model) + model + and VertexGeminiConfig._forward_gemini_function_call_id( + model, custom_llm_provider + ) ) if tool_calls is not None: @@ -1475,6 +1479,7 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], last_message_with_tool_calls: Optional[dict], model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> Union[VertexPartType, List[VertexPartType]]: """ OpenAI message with a tool result looks like: @@ -1616,14 +1621,16 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 name = tool.get("function", {}).get("name", "") # Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix). - # Only Gemini 3+ accepts (and returns) an `id` on function_response parts; - # older Gemini models reject the field with a 400. + # Only Google AI Studio Gemini 3+ accepts `id` on function_response parts. + # Vertex AI and older Gemini models reject the field with HTTP 400. from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) gemini_call_id: Optional[str] = None - if model and VertexGeminiConfig._is_gemini_3_or_newer(model): + if model and VertexGeminiConfig._forward_gemini_function_call_id( + model, custom_llm_provider + ): raw_tool_call_id = message.get("tool_call_id") if raw_tool_call_id and isinstance(raw_tool_call_id, str): stripped_id = raw_tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 72569e5c6c..10ad373152 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -151,4 +151,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): except Exception: # If conversion fails, leave as is and let the API handle it pass - return _gemini_convert_messages_with_history(messages=messages, model=model) + return _gemini_convert_messages_with_history( + messages=messages, + model=model, + custom_llm_provider="gemini", + ) diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index 950edbeb47..3d532113ba 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -174,7 +174,9 @@ def transform_openai_messages_to_gemini_context_caching( ) transformed_messages = _gemini_convert_messages_with_history( - messages=new_messages, model=model + messages=new_messages, + model=model, + custom_llm_provider=custom_llm_provider, ) model_name = "models/{}".format(model) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 44d31eb1b8..bec992b2bf 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -311,6 +311,7 @@ def check_if_part_exists_in_parts( def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> List[ContentType]: """ Converts given messages from OpenAI format to Gemini format @@ -548,7 +549,9 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( - assistant_msg, model=model + assistant_msg, + model=model, + custom_llm_provider=custom_llm_provider, ) ## check if gemini_tool_call already exists in assistant_content for gemini_tool_call_part in gemini_tool_call_parts: @@ -610,6 +613,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 messages[msg_i], # type: ignore last_message_with_tool_calls, # type: ignore model=model, + custom_llm_provider=custom_llm_provider, ) msg_i += 1 # Handle both single part and list of parts (for Computer Use with images) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 101828cbb5..0822b364a6 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -289,6 +289,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return True return False + @staticmethod + def _forward_gemini_function_call_id( + model: str, custom_llm_provider: Optional[str] = None + ) -> bool: + """ + Whether to include `id` on function_call / function_response parts. + + Gemini 3+ on Google AI Studio accepts (and returns) `id` for strict + tool-call matching. Vertex AI rejects the field with HTTP 400. + """ + if custom_llm_provider != "gemini": + return False + return VertexGeminiConfig._is_gemini_3_or_newer(model) + def _supports_penalty_parameters(self, model: str) -> bool: # Gemini 3 models do not support penalty parameters if VertexGeminiConfig._is_gemini_3_or_newer(model): @@ -2645,7 +2659,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): def _transform_messages( self, messages: List[AllMessageValues], model: Optional[str] = None ) -> List[ContentType]: - return _gemini_convert_messages_with_history(messages=messages, model=model) + return _gemini_convert_messages_with_history( + messages=messages, + model=model, + custom_llm_provider="vertex_ai", + ) def get_error_class( self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers] diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index b357b64156..a1d5397876 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -16,15 +16,15 @@ GeminiEmbeddingInput = Union[EmbeddingInput, List[List[str]]] class FunctionResponse(TypedDict, total=False): # `id` correlates this response with the originating `functionCall` part. - # Required by Gemini 3.5+ for strict function-calling response matching. + # Supported on Google AI Studio Gemini 3.5+; Vertex AI rejects this field. id: str name: Required[str] response: Optional[dict] class FunctionCall(TypedDict, total=False): - # `id` is returned by Gemini 3.5+ to correlate the corresponding - # `functionResponse`. Older Gemini models omit this field. + # `id` correlates the corresponding `functionResponse` on Google AI Studio + # Gemini 3.5+. Vertex AI and older Gemini models omit/reject this field. id: str name: Required[str] args: Optional[dict] @@ -52,8 +52,8 @@ class PartType(TypedDict, total=False): class HttpxFunctionCall(TypedDict, total=False): - # `id` is returned by Gemini 3.5+ to correlate the corresponding - # `functionResponse`. Older Gemini models omit this field. + # `id` correlates the corresponding `functionResponse` on Google AI Studio + # Gemini 3.5+. Vertex AI and older Gemini models omit/reject this field. id: str name: Required[str] args: dict diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 59c27f56bb..40ab4d1a89 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2097,6 +2097,125 @@ def test_is_gemini_3_or_newer(): assert VertexGeminiConfig._is_gemini_3_or_newer("") == False +def test_forward_gemini_function_call_id_vertex_vs_google_ai_studio(): + """Vertex AI rejects `id` on function_call/function_response; Google AI Studio accepts it on Gemini 3.5+.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + model = "gemini-3.5-flash" + assert ( + VertexGeminiConfig._forward_gemini_function_call_id(model, "vertex_ai") is False + ) + assert ( + VertexGeminiConfig._forward_gemini_function_call_id(model, "vertex_ai_beta") + is False + ) + assert VertexGeminiConfig._forward_gemini_function_call_id(model, "gemini") is True + assert VertexGeminiConfig._forward_gemini_function_call_id(model, None) is False + assert ( + VertexGeminiConfig._forward_gemini_function_call_id( + "gemini-2.5-flash", "gemini" + ) + is False + ) + + +def test_vertex_ai_gemini_35_tool_calls_omit_function_call_id(): + """Regression: Vertex must not send OpenAI tool_call id inside Gemini function_call parts.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + messages = [ + {"role": "user", "content": "Explore this directory"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_50e7e0fe0989464a89f188eda443", + "type": "function", + "function": { + "name": "read", + "arguments": '{"filePath": "/tmp"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_50e7e0fe0989464a89f188eda443", + "content": "ok", + }, + ] + + contents = _gemini_convert_messages_with_history( + messages=messages, + model="gemini-3.5-flash", + custom_llm_provider="vertex_ai", + ) + + for content in contents: + for part in content.get("parts", []): + fc = part.get("function_call") + if fc is not None: + assert "id" not in fc, f"Vertex payload must not include id: {fc}" + fr = part.get("function_response") + if fr is not None: + assert "id" not in fr, f"Vertex payload must not include id: {fr}" + + +def test_google_ai_studio_gemini_35_tool_calls_include_function_call_id(): + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + tool_call_id = "call_50e7e0fe0989464a89f188eda443" + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": tool_call_id, + "type": "function", + "function": { + "name": "read", + "arguments": '{"filePath": "/tmp"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": tool_call_id, + "content": "ok", + }, + ] + + contents = _gemini_convert_messages_with_history( + messages=messages, + model="gemini-3.5-flash", + custom_llm_provider="gemini", + ) + + function_call_ids = [] + function_response_ids = [] + for content in contents: + for part in content.get("parts", []): + fc = part.get("function_call") + if fc is not None: + function_call_ids.append(fc.get("id")) + fr = part.get("function_response") + if fr is not None: + function_response_ids.append(fr.get("id")) + + assert function_call_ids == [tool_call_id] + assert function_response_ids == [tool_call_id] + + def test_reasoning_effort_maps_to_thinking_level_gemini_3(): """Test that reasoning_effort maps to thinking_level AND includeThoughts for Gemini 3+ models""" from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -3531,7 +3650,12 @@ def test_video_metadata_supported_for_all_gemini_models(): } ] - for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro", "gemini-3-pro-preview"]: + for model in [ + "gemini-1.5-pro", + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-pro-preview", + ]: contents = _gemini_convert_messages_with_history(messages=messages, model=model) file_part = None @@ -3541,19 +3665,25 @@ def test_video_metadata_supported_for_all_gemini_models(): break assert file_part is not None, f"{model}: file part should exist" - assert "video_metadata" in file_part, f"{model}: video_metadata should be present" + assert ( + "video_metadata" in file_part + ), f"{model}: video_metadata should be present" assert file_part["video_metadata"]["fps"] == 5, f"{model}: fps should be 5" # Per-part media_resolution is Gemini 3+ only; 2.x uses generation_config global for model in ["gemini-3-pro-preview"]: contents = _gemini_convert_messages_with_history(messages=messages, model=model) file_part = next(p for p in contents[0]["parts"] if "file_data" in p) - assert "media_resolution" in file_part, f"{model}: media_resolution should be present" + assert ( + "media_resolution" in file_part + ), f"{model}: media_resolution should be present" for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro"]: contents = _gemini_convert_messages_with_history(messages=messages, model=model) file_part = next(p for p in contents[0]["parts"] if "file_data" in p) - assert "media_resolution" not in file_part, f"{model}: per-part media_resolution should not be set" + assert ( + "media_resolution" not in file_part + ), f"{model}: per-part media_resolution should not be set" def test_chunk_parser_handles_prompt_feedback_block(): @@ -4186,8 +4316,9 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): # DOCUMENT tokens should be included in text_tokens: 8 (TEXT) + 774 (DOCUMENT) = 782 assert result.prompt_tokens_details is not None - assert result.prompt_tokens_details.text_tokens == 782, \ - "DOCUMENT modality tokens should be added to text_tokens (8 TEXT + 774 DOCUMENT = 782)" + assert ( + result.prompt_tokens_details.text_tokens == 782 + ), "DOCUMENT modality tokens should be added to text_tokens (8 TEXT + 774 DOCUMENT = 782)" # Verify completion token details assert result.completion_tokens_details is not None @@ -4222,8 +4353,9 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): # DOCUMENT cached tokens map to cached_text_tokens, so: # text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382 - assert result.prompt_tokens_details.text_tokens == 382, \ - "text_tokens should be (8 + 774) - 400 cached = 382" + assert ( + result.prompt_tokens_details.text_tokens == 382 + ), "text_tokens should be (8 + 774) - 400 cached = 382" assert result.prompt_tokens_details.cached_tokens == 400