tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-04 18:27:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Day 0 support : Gemini 3.5 Flash (#28268)

Browse Source 
* Add day 0 support for gemini 3.5 flash

* Fix pricing

* Fix greptile review

* Fix failing test

* Fix tests

* Fix: revert tool removing logic

* fix greptile and test

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
(cherry picked from commit 3c3d131f01)
This commit is contained in:
Sameer Kankute
2026-05-20 18:49:58 -07:00
committed by Yuneng Jiang
parent e58a561caa
commit cbf9ffec30
8 changed files with 576 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/litellm_core_utils/prompt_templates/factory.py
+35 -5
View File
@@ -1233,6 +1233,7 @@ def infer_protocol_value(
def _gemini_tool_call_invoke_helper(
function_call_params: ChatCompletionToolCallFunctionChunk,
tool_call_id: Optional[str] = None,
) -> Optional[VertexFunctionCall]:
name = function_call_params.get("name", "") or ""
arguments = function_call_params.get("arguments", "")
@@ -1248,6 +1249,10 @@ def _gemini_tool_call_invoke_helper(
name=name,
args=arguments_dict,
)
if tool_call_id:
clean_id = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
if clean_id:
function_call["id"] = clean_id
return function_call
@@ -1384,12 +1389,23 @@ def convert_to_gemini_tool_call_invoke(
tool_calls = message.get("tool_calls", None)
function_call = message.get("function_call", None)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
forward_tool_call_id = bool(
model and VertexGeminiConfig._is_gemini_3_or_newer(model)
)
if tool_calls is not None:
for idx, tool in enumerate(tool_calls):
if "function" in tool:
gemini_function_call: Optional[VertexFunctionCall] = (
_gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
function_call_params=tool["function"],
tool_call_id=(
tool.get("id") if forward_tool_call_id else None
),
)
)
if gemini_function_call is not None:
@@ -1429,10 +1445,6 @@ def convert_to_gemini_tool_call_invoke(
thought_signature = provider_fields.get("thought_signature")
# If no signature found and model is gemini-3, use dummy signature
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
if (
not thought_signature
and model
@@ -1462,6 +1474,7 @@ def convert_to_gemini_tool_call_invoke(
def convert_to_gemini_tool_call_result( # noqa: PLR0915
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
last_message_with_tool_calls: Optional[dict],
model: Optional[str] = None,
) -> Union[VertexPartType, List[VertexPartType]]:
"""
OpenAI message with a tool result looks like:
@@ -1602,6 +1615,21 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
):
name = tool.get("function", {}).get("name", "")
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
# Only Gemini 3+ accepts (and returns) an `id` on function_response parts;
# older Gemini models reject the field with a 400.
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
gemini_call_id: Optional[str] = None
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
raw_tool_call_id = message.get("tool_call_id")
if raw_tool_call_id and isinstance(raw_tool_call_id, str):
stripped_id = raw_tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
if stripped_id:
gemini_call_id = stripped_id
if not name:
raise Exception(
"Missing corresponding tool call for tool response message. Received - message={}, last_message_with_tool_calls={}".format(
@@ -1632,6 +1660,8 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
name=name,
response=response_data, # type: ignore
)
if gemini_call_id:
_function_response["id"] = gemini_call_id
# Create part with function_response, and optionally inline_data for images (Computer Use)
_part: VertexPartType = {"function_response": _function_response}
litellm/llms/vertex_ai/gemini/transformation.py
+3 -1
View File
@@ -607,7 +607,9 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
and messages[msg_i]["role"] in tool_call_message_roles
):
_part = convert_to_gemini_tool_call_result(
messages[msg_i], last_message_with_tool_calls # type: ignore
messages[msg_i], # type: ignore
last_message_with_tool_calls, # type: ignore
model=model,
)
msg_i += 1
# Handle both single part and list of parts (for Computer Use with images)
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+119 -9
View File
@@ -280,6 +280,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
- gemini-3-pro-preview
- gemini-3-flash
- gemini-3-flash-preview (Gemini 3 Flash)
- gemini-3.1-pro-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview
- gemini-3.5-flash
- Any future Gemini 3.x models
"""
# Check for Gemini 3 models
@@ -300,6 +302,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
supported_params = [
"temperature",
"top_p",
"top_k",
"max_tokens",
"max_completion_tokens",
"stream",
@@ -363,6 +366,66 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"""
return Tools(googleSearch={})
@staticmethod
def _search_tool_keys() -> set:
return {
VertexToolName.GOOGLE_SEARCH.value,
VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value,
VertexToolName.ENTERPRISE_WEB_SEARCH.value,
VertexToolName.URL_CONTEXT.value,
"google_search",
"google_search_retrieval",
"enterprise_web_search",
"urlContext",
}
@classmethod
def _drop_search_tools_mixed_with_functions(cls, optional_params: dict) -> None:
"""
Drop search tools from optional_params when mixed with function declarations
and include_server_side_tool_invocations is not enabled.
Runs after map_openai_params merges tools and web_search_options so both
code paths (single _map_function call vs split tools + web_search_options)
get the same conflict resolution.
"""
if optional_params.get("include_server_side_tool_invocations"):
return
tools = optional_params.get("tools")
if not isinstance(tools, list) or not tools:
return
search_tool_keys = cls._search_tool_keys()
has_function_declarations = any(
isinstance(tool, dict) and tool.get("function_declarations")
for tool in tools
)
if not has_function_declarations:
return
has_search_tools = any(
isinstance(tool, dict) and any(key in tool for key in search_tool_keys)
for tool in tools
)
if not has_search_tools:
return
verbose_logger.warning(
"Vertex AI does not support mixing function declarations with "
"search tools (googleSearch, enterpriseWebSearch, urlContext, "
"googleSearchRetrieval) in the same request. Dropping search "
"tools and keeping function declarations. To use search tools, "
"send a request without function calling tools."
)
optional_params["tools"] = [
tool
for tool in tools
if not (
isinstance(tool, dict) and any(key in tool for key in search_tool_keys)
)
]
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
"""
Map OpenAI service_tier (string) to Gemini serviceTier.
@@ -884,9 +947,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
GeminiThinkingConfig with thinkingLevel and includeThoughts
"""
# Check if this is gemini-3-flash which supports MINIMAL thinking level
# Covers gemini-3-flash, gemini-3-flash-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview, etc.
# Covers gemini-3-flash, gemini-3-flash-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview,
# gemini-3.5-flash, and any future 3.x-flash variants.
is_gemini3flash = model and (
"gemini-3-flash" in model.lower() or "gemini-3.1-flash" in model.lower()
"flash" in model.lower() and "gemini-3" in model.lower()
)
is_gemini31pro = model and ("gemini-3.1-pro-preview" in model.lower())
if reasoning_effort == "minimal":
@@ -982,8 +1046,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# Follow provider defaults unless explicitly opted into legacy behavior.
if litellm.enable_gemini_default_thinking_level_low is True:
is_gemini3flash = (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
"gemini-3" in model.lower() and "flash" in model.lower()
)
params["thinkingLevel"] = (
"minimal" if is_gemini3flash else "low"
@@ -1077,6 +1140,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model: str,
drop_params: bool,
) -> Dict:
gemini_sampling_params_warned: bool = False
for param, value in non_default_params.items():
if param == "temperature":
if VertexGeminiConfig._is_gemini_3_or_newer(model):
@@ -1086,9 +1150,41 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"can cause infinite loops, degraded reasoning performance, and failure on complex tasks. "
"Strongly recommended to use temperature = 1.0 (default)."
)
if not gemini_sampling_params_warned:
verbose_logger.warning(
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to "
f"function for Gemini 3+ ({model}) but are planned for removal in a "
"future release. Move sampling guidance into the `system` "
"instructions instead."
)
gemini_sampling_params_warned = True
optional_params["temperature"] = value
elif param == "top_p":
if (
VertexGeminiConfig._is_gemini_3_or_newer(model)
and not gemini_sampling_params_warned
):
verbose_logger.warning(
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to "
f"function for Gemini 3+ ({model}) but are planned for removal in a "
"future release. Move sampling guidance into the `system` "
"instructions instead."
)
gemini_sampling_params_warned = True
optional_params["top_p"] = value
elif param == "top_k":
if (
VertexGeminiConfig._is_gemini_3_or_newer(model)
and not gemini_sampling_params_warned
):
verbose_logger.warning(
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to "
f"function for Gemini 3+ ({model}) but are planned for removal in a "
"future release. Move sampling guidance into the `system` "
"instructions instead."
)
gemini_sampling_params_warned = True
optional_params["top_k"] = value
elif (
param == "stream" and value is True
): # sending stream = False, can cause it to get passed unchecked and raise issues
@@ -1139,11 +1235,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
elif param == "parallel_tool_calls":
if value is False and not (
drop_params or litellm.drop_params
): # if drop params is True, then we should just ignore this
self.validate_parallel_tool_calls(value, non_default_params)
else:
tools_list = non_default_params.get(
"tools", non_default_params.get("functions")
)
num_tools = len(tools_list) if isinstance(tools_list, list) else 0
# Gemini does not support parallel_tool_calls=False with multiple
# tools. Drop the param instead of failing — Responses API clients
# often send parallel_tool_calls=false by default.
if not (value is False and num_tools > 1):
optional_params["parallel_tool_calls"] = value
elif param == "seed":
optional_params["seed"] = value
@@ -1216,6 +1315,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if "temperature" not in optional_params:
optional_params["temperature"] = 1.0
self._drop_search_tools_mixed_with_functions(optional_params)
return optional_params
def get_mapped_special_auth_params(self) -> dict:
@@ -1588,6 +1689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
}
# Extract thought signature if present
thought_signature = part.get("thoughtSignature")
# Gemini 3.5+ returns a stable `id` per function call to enable
# strict response matching. Preserve it as the OpenAI
# tool_call_id so it can be echoed back unchanged.
gemini_call_id = part["functionCall"].get("id")
if is_function_call is True:
function_dict: Dict[str, Any] = dict(_function_chunk)
@@ -1605,6 +1710,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"function": _function_chunk,
"index": cumulative_tool_call_idx,
}
# Gemini 3.5+ returns a stable native `id`; prefer it over
# the synthetic call_<uuid> so the same value can be echoed
# back on the matching `functionResponse`.
if gemini_call_id:
_tool_response_chunk["id"] = gemini_call_id
# Embed thought signature in ID for OpenAI client compatibility
if thought_signature:
_tool_response_chunk["provider_specific_fields"] = { # type: ignore
litellm/model_prices_and_context_window_backup.json
+178
View File
@@ -15237,6 +15237,64 @@
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"input_cost_per_audio_token": 1e-06,
"litellm_provider": "vertex_ai",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -16614,6 +16672,67 @@
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"rpm": 2000,
"source": "https://ai.google.dev/pricing/gemini-3",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 800000,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -16799,6 +16918,65 @@
},
"web_search_billing_unit": "per_query"
},
"gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"source": "https://ai.google.dev/pricing/gemini-3",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
litellm/types/llms/vertex_ai.py
+15 -6
View File
@@ -14,13 +14,19 @@ from litellm.types.llms.openai import EmbeddingInput
GeminiEmbeddingInput = Union[EmbeddingInput, List[List[str]]]
class FunctionResponse(TypedDict):
name: str
class FunctionResponse(TypedDict, total=False):
# `id` correlates this response with the originating `functionCall` part.
# Required by Gemini 3.5+ for strict function-calling response matching.
id: str
name: Required[str]
response: Optional[dict]
class FunctionCall(TypedDict):
name: str
class FunctionCall(TypedDict, total=False):
# `id` is returned by Gemini 3.5+ to correlate the corresponding
# `functionResponse`. Older Gemini models omit this field.
id: str
name: Required[str]
args: Optional[dict]
@@ -45,8 +51,11 @@ class PartType(TypedDict, total=False):
media_resolution: Literal["low", "medium", "high"]
class HttpxFunctionCall(TypedDict):
name: str
class HttpxFunctionCall(TypedDict, total=False):
# `id` is returned by Gemini 3.5+ to correlate the corresponding
# `functionResponse`. Older Gemini models omit this field.
id: str
name: Required[str]
args: dict
model_prices_and_context_window.json
+178
View File
@@ -15242,6 +15242,64 @@
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"input_cost_per_audio_token": 1e-06,
"litellm_provider": "vertex_ai",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -16619,6 +16677,67 @@
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"rpm": 2000,
"source": "https://ai.google.dev/pricing/gemini-3",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 800000,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -16804,6 +16923,65 @@
},
"web_search_billing_unit": "per_query"
},
"gemini-3.5-flash": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_pdf_size_mb": 30,
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"source": "https://ai.google.dev/pricing/gemini-3",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 2.7e-06,
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"supports_service_tier": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
+32
View File
@@ -2959,6 +2959,38 @@ def test_vertex_ai_gemini3_tool_combination_no_drop():
assert len(tools) == 3
def test_vertex_ai_mixed_tools_and_web_search_options_drops_search():
"""
When function tools and web_search_options are sent separately (Codex-style),
search tools are dropped unless include_server_side_tool_invocations is set.
"""
v = VertexGeminiConfig()
optional_params: dict = {}
non_default_params = {
"tools": [
{
"type": "function",
"function": {"name": "exec_command", "description": "Run a command"},
}
],
"web_search_options": {},
}
result = v.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="gemini-3.5-flash",
drop_params=True,
)
assert not result.get("include_server_side_tool_invocations")
tool_keys = set()
for tool in result.get("tools", []):
tool_keys.update(tool.keys())
assert "function_declarations" in tool_keys
assert "googleSearch" not in tool_keys
def test_vertex_ai_openai_web_search_tool_transformation():
"""
Test that OpenAI-style web_search and web_search_preview tools are transformed to googleSearch.
tests/test_litellm/llms/vertex_ai/test_vertex.py
+16 -24
View File
@@ -1490,39 +1490,31 @@ def test_vertex_parallel_tool_calls_true():
assert "tools" in optional_params
def test_vertex_parallel_tool_calls_false_multiple_tools_error():
def test_vertex_parallel_tool_calls_false_multiple_tools_dropped():
"""
Test that parallel_tool_calls = False with multiple tools raises UnsupportedParamsError
when drop_params is False.
parallel_tool_calls=False with multiple tools is dropped for Gemini
(unsupported upstream). Request should succeed without the param.
"""
tools = [
{"type": "function", "function": {"name": "get_weather"}},
{"type": "function", "function": {"name": "get_time"}},
]
with pytest.raises(litellm.utils.UnsupportedParamsError) as excinfo:
get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
parallel_tool_calls=False,
)
assert (
"`parallel_tool_calls=False` is not supported by Gemini when multiple tools are"
in str(excinfo.value)
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
parallel_tool_calls=False,
)
assert "parallel_tool_calls" not in optional_params
assert "tools" in optional_params
# works when specified as "functions"
with pytest.raises(litellm.utils.UnsupportedParamsError) as excinfo:
get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
functions=tools,
parallel_tool_calls=False,
)
assert (
"`parallel_tool_calls=False` is not supported by Gemini when multiple tools are"
in str(excinfo.value)
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
functions=tools,
parallel_tool_calls=False,
)
assert "parallel_tool_calls" not in optional_params
def test_vertex_parallel_tool_calls_false_single_tool():