fix(gemini): googleSearch + server-side tools and googleMaps JSON schema (#29582)

* fix(gemini): keep googleSearch with server-side tools and googleMaps JSON schema

Wire include_server_side_tool_invocations through completion() so mixed
google_search and function tools are not dropped on Gemini 3+. Rewrite
generationConfig to responseFormat when googleMaps is used with JSON schema.

Fixes #27479
Fixes #29451

Co-authored-by: Cursor <cursoragent@cursor.com>

* address greptile review feedback (greploop iteration 1)

* style: fix black formatting in main.py for py312 compat

* Fix Gemini Google Maps extra_body JSON rewrite

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

litellm/constants.py

@@ -678,6 +678,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [
     "extra_headers",
     "thinking",
     "web_search_options",
+    "include_server_side_tool_invocations",
     "service_tier",
     "prompt_cache_key",
     "prompt_cache_retention",
@@ -739,6 +740,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
     "verbosity": None,
     "thinking": None,
     "web_search_options": None,
+    "include_server_side_tool_invocations": None,
     "service_tier": None,
     "safety_identifier": None,
     "prompt_cache_key": None,

litellm/llms/gemini/chat/transformation.py

@@ -93,6 +93,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
             "modalities",
             "parallel_tool_calls",
             "web_search_options",
+            "include_server_side_tool_invocations",
             "service_tier",
         ]
         if supports_reasoning(model, custom_llm_provider="gemini"):

litellm/llms/vertex_ai/gemini/transformation.py

@@ -1121,6 +1121,61 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
                 data_dict[k] = v
 
 
+def _has_google_maps_tool(tools: Optional[Any]) -> bool:
+    """Return True if any tool object in the list has a 'googleMaps' key."""
+    if not isinstance(tools, list):
+        return False
+    return any(
+        isinstance(t, dict) and VertexToolName.GOOGLE_MAPS.value in t for t in tools
+    )
+
+
+def _rewrite_mime_type_to_response_format(generation_config: GenerationConfig) -> None:
+    """
+    Convert response_mime_type + response_json_schema/response_schema to the newer
+    responseFormat structure when googleMaps is present in tools.
+
+    The Gemini API rejects the combination of googleMaps + response_mime_type:
+    'application/json' with the error:
+        "Google Maps tool with a response mime type: 'application/json' is unsupported"
+
+    The newer responseFormat field supports this combination on both the Gemini API
+    (generativelanguage.googleapis.com) and Vertex AI endpoints.
+
+    Before:
+        generationConfig: {
+            response_mime_type: "application/json",
+            response_json_schema: {...}
+        }
+
+    After:
+        generationConfig: {
+            responseFormat: {
+                "text": {"mimeType": "APPLICATION_JSON", "schema": {...}}
+            }
+        }
+    """
+    schema = generation_config.pop("response_json_schema", None)  # type: ignore[misc]
+    if schema is None:
+        schema = generation_config.pop("response_schema", None)  # type: ignore[misc]
+    generation_config.pop("response_mime_type", None)  # type: ignore[misc]
+
+    response_format: Dict[str, Any] = {"text": {"mimeType": "APPLICATION_JSON"}}
+    if schema is not None:
+        response_format["text"]["schema"] = schema
+    generation_config["responseFormat"] = response_format  # type: ignore[typeddict-unknown-key]
+
+
+def _rewrite_google_maps_response_format(data: RequestBody) -> None:
+    generation_config = cast(Optional[GenerationConfig], data.get("generationConfig"))
+    if (
+        isinstance(generation_config, dict)
+        and _has_google_maps_tool(data.get("tools"))
+        and generation_config.get("response_mime_type") == "application/json"
+    ):
+        _rewrite_mime_type_to_response_format(generation_config)
+
+
 def _transform_request_body(  # noqa: PLR0915
     messages: List[AllMessageValues],
     model: str,
@@ -1246,6 +1301,7 @@ def _transform_request_body(  # noqa: PLR0915
         if labels and custom_llm_provider != LlmProviders.GEMINI:
             data["labels"] = labels
         _pop_and_merge_extra_body(data, optional_params)
+        _rewrite_google_maps_response_format(data)
     except Exception as e:
         raise e
 

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -1147,6 +1147,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return cast(dict, speech_config)
 
+    @staticmethod
+    def _apply_include_server_side_tool_invocations(
+        non_default_params: Dict,
+        optional_params: Dict,
+    ) -> None:
+        """
+        Set include_server_side_tool_invocations before tools are mapped.
+
+        map_openai_params iterates non_default_params in request order; if tools
+        appear before this flag, _resolve_search_tool_conflict would drop search
+        tools before the flag is applied.
+        """
+        for key in (
+            "include_server_side_tool_invocations",
+            "includeServerSideToolInvocations",
+        ):
+            if non_default_params.get(key) is True or optional_params.get(key) is True:
+                optional_params["include_server_side_tool_invocations"] = True
+                return
+
     def map_openai_params(  # noqa: PLR0915
         self,
         non_default_params: Dict,
@@ -1154,6 +1174,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         model: str,
         drop_params: bool,
     ) -> Dict:
+        self._apply_include_server_side_tool_invocations(
+            non_default_params, optional_params
+        )
         gemini_sampling_params_warned: bool = False
         for param, value in non_default_params.items():
             if param == "temperature":

litellm/main.py

@@ -437,6 +437,7 @@ async def acompletion(  # noqa: PLR0915
     # Optional liteLLM function params
     thinking: Optional[AnthropicThinkingParam] = None,
     web_search_options: Optional[OpenAIWebSearchOptions] = None,
+    include_server_side_tool_invocations: Optional[bool] = None,
     # Session management
     shared_session: Optional["ClientSession"] = None,
     # Per-request JSON schema validation (overrides litellm.enable_json_schema_validation)
@@ -584,6 +585,7 @@ async def acompletion(  # noqa: PLR0915
         "acompletion": True,  # assuming this is a required parameter
         "thinking": thinking,
         "web_search_options": web_search_options,
+        "include_server_side_tool_invocations": include_server_side_tool_invocations,
         "shared_session": shared_session,
         "enable_json_schema_validation": enable_json_schema_validation,
     }
@@ -1116,6 +1118,7 @@ def completion(  # type: ignore # noqa: PLR0915
     top_logprobs: Optional[int] = None,
     parallel_tool_calls: Optional[bool] = None,
     web_search_options: Optional[OpenAIWebSearchOptions] = None,
+    include_server_side_tool_invocations: Optional[bool] = None,
     deployment_id=None,
     extra_headers: Optional[dict] = None,
     safety_identifier: Optional[str] = None,
@@ -1550,6 +1553,11 @@ def completion(  # type: ignore # noqa: PLR0915
             "reasoning_effort": reasoning_effort,
             "thinking": thinking,
             "web_search_options": web_search_options,
+            "include_server_side_tool_invocations": (
+                include_server_side_tool_invocations
+                if include_server_side_tool_invocations is not None
+                else kwargs.get("include_server_side_tool_invocations")
+            ),
             "safety_identifier": safety_identifier,
             "service_tier": service_tier,
             "allowed_openai_params": kwargs.get("allowed_openai_params"),

litellm/types/llms/vertex_ai.py

@@ -246,6 +246,7 @@ class GenerationConfig(TypedDict, total=False):
     response_mime_type: Literal["text/plain", "application/json"]
     response_schema: dict
     response_json_schema: dict
+    responseFormat: dict
     seed: int
     responseLogprobs: bool
     logprobs: int

tests/litellm/llms/vertex_ai/gemini/test_transformation.py

@@ -246,6 +246,38 @@ async def test__transform_request_body_image_config_with_image_size():
     assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K"
 
 
+def test__transform_request_body_google_maps_json_schema_uses_response_format():
+    """googleMaps + JSON schema must use responseFormat, not response_mime_type."""
+    messages = [{"role": "user", "content": "Find restaurants in Mumbai"}]
+    schema = {
+        "type": "object",
+        "properties": {"places": {"type": "array"}},
+        "required": ["places"],
+    }
+    optional_params = {
+        "tools": [{"googleMaps": {}}],
+        "response_mime_type": "application/json",
+        "response_json_schema": schema,
+    }
+    transform_request_params = {
+        "messages": messages,
+        "model": "gemini/gemini-3.1-flash-lite",
+        "optional_params": optional_params,
+        "custom_llm_provider": "gemini",
+        "litellm_params": {},
+        "cached_content": None,
+    }
+
+    rb: RequestBody = transformation._transform_request_body(**transform_request_params)
+
+    gen = rb["generationConfig"]
+    assert "responseFormat" in gen
+    assert gen["responseFormat"]["text"]["mimeType"] == "APPLICATION_JSON"
+    assert gen["responseFormat"]["text"]["schema"] == schema
+    assert "response_mime_type" not in gen
+    assert "response_json_schema" not in gen
+
+
 def test_map_function_google_search_snake_case():
     """
     Test that google_search tool (snake_case) is properly mapped to googleSearch.

tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py

@@ -285,6 +285,80 @@ def test_extra_body_tags_not_forwarded_to_vertex_ai():
     assert result["custom_param"] == "allowed"
 
 
+def test_extra_body_google_maps_rewrites_json_response_format():
+    messages = [{"role": "user", "content": "test"}]
+    optional_params = {
+        "response_mime_type": "application/json",
+        "response_schema": {
+            "type": "object",
+            "properties": {"answer": {"type": "string"}},
+        },
+        "extra_body": {
+            "tools": [{"googleMaps": {}}],
+        },
+    }
+
+    result = _transform_request_body(
+        messages=messages,
+        model="gemini-2.5-pro",
+        optional_params=optional_params,
+        custom_llm_provider="vertex_ai",
+        litellm_params={},
+        cached_content=None,
+    )
+
+    generation_config = result["generationConfig"]
+    assert "response_mime_type" not in generation_config
+    assert generation_config["responseFormat"] == {
+        "text": {
+            "mimeType": "APPLICATION_JSON",
+            "schema": {
+                "type": "object",
+                "properties": {"answer": {"type": "string"}},
+            },
+        }
+    }
+
+
+def test_extra_body_generation_config_cannot_restore_google_maps_json_mime_type():
+    messages = [{"role": "user", "content": "test"}]
+    optional_params = {
+        "tools": [{"googleMaps": {}}],
+        "response_mime_type": "application/json",
+        "extra_body": {
+            "generationConfig": {
+                "response_mime_type": "application/json",
+                "response_json_schema": {
+                    "type": "object",
+                    "properties": {"answer": {"type": "string"}},
+                },
+            },
+        },
+    }
+
+    result = _transform_request_body(
+        messages=messages,
+        model="gemini-2.5-pro",
+        optional_params=optional_params,
+        custom_llm_provider="vertex_ai",
+        litellm_params={},
+        cached_content=None,
+    )
+
+    generation_config = result["generationConfig"]
+    assert "response_mime_type" not in generation_config
+    assert "response_json_schema" not in generation_config
+    assert generation_config["responseFormat"] == {
+        "text": {
+            "mimeType": "APPLICATION_JSON",
+            "schema": {
+                "type": "object",
+                "properties": {"answer": {"type": "string"}},
+            },
+        }
+    }
+
+
 def test_metadata_to_labels_vertex_only():
     """Test that metadata->labels conversion only happens for Vertex AI"""
     messages = [{"role": "user", "content": "test"}]

tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py

@@ -3078,6 +3078,83 @@ def test_vertex_ai_gemini3_tool_combination_no_drop():
     assert len(tools) == 3
 
 
+def test_get_optional_params_keeps_google_search_with_server_side_flag():
+    """
+    include_server_side_tool_invocations must be in non_default_params before
+    map_openai_params runs (not only via add_provider_specific_params after).
+    """
+    from litellm.utils import get_optional_params
+
+    optional_params = get_optional_params(
+        model="gemini-3.1-pro-preview",
+        custom_llm_provider="gemini",
+        tools=[
+            {"google_search": {}},
+            {
+                "type": "function",
+                "function": {
+                    "name": "send_message",
+                    "description": "Send a message back",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"message": {"type": "string"}},
+                        "required": ["message"],
+                    },
+                },
+            },
+        ],
+        include_server_side_tool_invocations=True,
+    )
+
+    assert optional_params.get("include_server_side_tool_invocations") is True
+    tool_keys = set()
+    for tool in optional_params.get("tools", []):
+        tool_keys.update(tool.keys())
+    assert "function_declarations" in tool_keys
+    assert "googleSearch" in tool_keys
+
+
+def test_map_openai_params_tools_before_include_server_side_flag():
+    """
+    Request bodies often list tools before include_server_side_tool_invocations.
+    Search tools must not be dropped when the flag is present later in the dict.
+    """
+    v = VertexGeminiConfig()
+    optional_params: dict = {}
+    non_default_params = {
+        "tools": [
+            {"google_search": {}},
+            {
+                "type": "function",
+                "function": {
+                    "name": "send_message",
+                    "description": "Send a message back",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"message": {"type": "string"}},
+                        "required": ["message"],
+                    },
+                },
+            },
+        ],
+        "include_server_side_tool_invocations": True,
+    }
+
+    result = v.map_openai_params(
+        non_default_params=non_default_params,
+        optional_params=optional_params,
+        model="gemini-3.1-pro-preview",
+        drop_params=True,
+    )
+
+    assert result.get("include_server_side_tool_invocations") is True
+    tool_keys = set()
+    for tool in result.get("tools", []):
+        tool_keys.update(tool.keys())
+    assert "function_declarations" in tool_keys
+    assert "googleSearch" in tool_keys
+
+
 def test_vertex_ai_mixed_tools_and_web_search_options_drops_search():
     """
     When function tools and web_search_options are sent separately (Codex-style),