diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 5055b5db5a..2613415351 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -532,6 +532,12 @@ def convert_to_model_response_object( # noqa: PLR0915 if finish_reason is None: # gpt-4 vision can return 'finish_reason' or 'finish_details' finish_reason = choice.get("finish_details") or "stop" + if ( + finish_reason == "stop" + and message.tool_calls + and len(message.tool_calls) > 0 + ): + finish_reason = "tool_calls" logprobs = choice.get("logprobs", None) enhancements = choice.get("enhancements", None) choice = Choices( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 5e665a6e25..c94b276a7b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,17 @@ import io import mimetypes import re from os import PathLike -from typing import Any, Dict, List, Literal, Mapping, Optional, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Mapping, + Optional, + Union, + cast, +) from litellm.types.llms.openai import ( AllMessageValues, @@ -25,6 +35,9 @@ from litellm.types.utils import ( StreamingChoices, ) +if TYPE_CHECKING: # newer pattern to avoid importing pydantic objects on __init__.py + from litellm.types.llms.openai import ChatCompletionImageObject + DEFAULT_USER_CONTINUE_MESSAGE = ChatCompletionUserMessage( content="Please continue.", role="user" ) @@ -603,3 +616,57 @@ def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: if file_id: file_ids.append(file_id) return file_ids + + +def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: + """ + Filters a value from a dictionary + + Goes through the nested dict and removes the key if it exists + """ + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH + + if depth > DEFAULT_MAX_RECURSE_DEPTH: + return dictionary + + # Create a copy of keys to avoid modifying dict during iteration + keys = list(dictionary.keys()) + for k in keys: + v = dictionary[k] + if k == key: + del dictionary[k] + elif isinstance(v, dict): + filter_value_from_dict(v, key, depth + 1) + elif isinstance(v, list): + for item in v: + if isinstance(item, dict): + filter_value_from_dict(item, key, depth + 1) + return dictionary + + +def migrate_file_to_image_url( + message: "ChatCompletionFileObject", +) -> "ChatCompletionImageObject": + """ + Migrate file to image_url + """ + from litellm.types.llms.openai import ( + ChatCompletionImageObject, + ChatCompletionImageUrlObject, + ) + + file_id = message["file"].get("file_id") + file_data = message["file"].get("file_data") + format = message["file"].get("format") + if not file_id and not file_data: + raise ValueError("file_id and file_data are both None") + image_url_object = ChatCompletionImageObject( + type="image_url", + image_url=ChatCompletionImageUrlObject( + url=cast(str, file_id or file_data), + ), + ) + if format and isinstance(image_url_object["image_url"], dict): + image_url_object["image_url"]["format"] = format + return image_url_object + diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 2a795bdf2f..3bb1b5f378 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -186,11 +186,24 @@ class FireworksAIConfig(OpenAIGPTConfig): """ Add 'transform=inline' to the url of the image_url """ + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + filter_value_from_dict, + migrate_file_to_image_url, + ) + disable_add_transform_inline_image_block = cast( Optional[bool], litellm_params.get("disable_add_transform_inline_image_block") or litellm.disable_add_transform_inline_image_block, ) + ## For any 'file' message type with pdf content, move to 'image_url' message type + for message in messages: + if message["role"] == "user": + _message_content = message.get("content") + if _message_content is not None and isinstance(_message_content, list): + for idx, content in enumerate(_message_content): + if content["type"] == "file": + _message_content[idx] = migrate_file_to_image_url(content) for message in messages: if message["role"] == "user": _message_content = message.get("content") @@ -202,6 +215,8 @@ class FireworksAIConfig(OpenAIGPTConfig): model=model, disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, ) + filter_value_from_dict(cast(dict, message), "cache_control") + return messages def get_provider_info(self, model: str) -> ProviderSpecificModelInfo: diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 743ad73cf3..375ac089a0 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -84,6 +84,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "frequency_penalty", "modalities", "parallel_tool_calls", + "web_search_options", ] if supports_reasoning(model): supported_params.append("reasoning_effort") diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 1ab851b9b4..4b1c1fc620 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -220,6 +220,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "top_logprobs", "modalities", "parallel_tool_calls", + "web_search_options", ] if supports_reasoning(model): supported_params.append("reasoning_effort") @@ -251,6 +252,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): status_code=400, ) + def _map_web_search_options(self, value: dict) -> Tools: + """ + Base Case: empty dict + + Google doesn't support user_location or search_context_size params + """ + return Tools(googleSearch={}) + def _map_function(self, value: List[dict]) -> List[Tools]: gtool_func_declarations = [] googleSearch: Optional[dict] = None @@ -445,6 +454,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_modalities.append("MODALITY_UNSPECIFIED") return response_modalities + def validate_parallel_tool_calls(self, value: bool, non_default_params: dict): + tools = non_default_params.get("tools", non_default_params.get("functions")) + num_function_declarations = len(tools) if isinstance(tools, list) else 0 + if num_function_declarations > 1: + raise litellm.utils.UnsupportedParamsError( + message=( + "`parallel_tool_calls=False` is not supported by Gemini when multiple tools are " + "provided. Specify a single tool, or set " + "`parallel_tool_calls=True`. If you want to drop this param, set `litellm.drop_params = True` or pass in `(.., drop_params=True)` in the requst - https://docs.litellm.ai/docs/completion/drop_params" + ), + status_code=400, + ) + def map_openai_params( self, non_default_params: Dict, @@ -487,7 +509,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): and isinstance(value, list) and value ): - optional_params["tools"] = self._map_function(value=value) + optional_params = self._add_tools_to_optional_params( + optional_params, self._map_function(value=value) + ) elif param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): @@ -500,21 +524,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if value is False and not ( drop_params or litellm.drop_params ): # if drop params is True, then we should just ignore this - tools = non_default_params.get( - "tools", non_default_params.get("functions") - ) - num_function_declarations = ( - len(tools) if isinstance(tools, list) else 0 - ) - if num_function_declarations > 1: - raise litellm.utils.UnsupportedParamsError( - message=( - "`parallel_tool_calls=False` is not supported when multiple tools are " - "provided for Gemini. Specify a single tool, or set " - "`parallel_tool_calls=True`. If you want to drop this param, set `litellm.drop_params = True` or pass in `(.., drop_params=True)` in the requst - https://docs.litellm.ai/docs/completion/drop_params" - ), - status_code=400, - ) + self.validate_parallel_tool_calls(value, non_default_params) else: optional_params["parallel_tool_calls"] = value elif param == "seed": @@ -532,7 +542,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif param == "modalities" and isinstance(value, list): response_modalities = self.map_response_modalities(value) optional_params["responseModalities"] = response_modalities - + elif param == "web_search_options" and value and isinstance(value, dict): + _tools = self._map_web_search_options(value) + optional_params = self._add_tools_to_optional_params( + optional_params, [_tools] + ) if litellm.vertex_ai_safety_settings is not None: optional_params["safety_settings"] = litellm.vertex_ai_safety_settings return optional_params diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 804abe30f0..c277b9623c 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -3,6 +3,7 @@ from typing import List, Optional, Tuple import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( + filter_value_from_dict, strip_name_from_messages, ) from litellm.secret_managers.main import get_secret_str @@ -44,6 +45,7 @@ class XAIChatConfig(OpenAIGPTConfig): "top_logprobs", "top_p", "user", + "web_search_options", ] try: if litellm.supports_reasoning( @@ -66,6 +68,14 @@ class XAIChatConfig(OpenAIGPTConfig): for param, value in non_default_params.items(): if param == "max_completion_tokens": optional_params["max_tokens"] = value + elif param == "tools" and value is not None: + tools = [] + for tool in value: + tool = filter_value_from_dict(tool, "strict") + if tool is not None: + tools.append(tool) + if len(tools) > 0: + optional_params["tools"] = tools elif param in supported_openai_params: if value is not None: optional_params[param] = value diff --git a/litellm/llms/xai/common_utils.py b/litellm/llms/xai/common_utils.py index a26dc1e043..df324cf3ee 100644 --- a/litellm/llms/xai/common_utils.py +++ b/litellm/llms/xai/common_utils.py @@ -6,9 +6,21 @@ import litellm from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ProviderSpecificModelInfo class XAIModelInfo(BaseLLMModelInfo): + def get_provider_info( + self, + model: str, + ) -> Optional[ProviderSpecificModelInfo]: + """ + Default values all models of this provider support. + """ + return { + "supports_web_search": True, + } + def validate_environment( self, headers: dict, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 988000c640..4432be88b6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -96,13 +96,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_native_streaming": true }, "gpt-4.1-2025-04-14": { "max_tokens": 32768, @@ -135,13 +129,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_native_streaming": true }, "gpt-4.1-mini": { "max_tokens": 32768, @@ -174,13 +162,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_native_streaming": true }, "gpt-4.1-mini-2025-04-14": { "max_tokens": 32768, @@ -213,13 +195,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_native_streaming": true }, "gpt-4.1-nano": { "max_tokens": 32768, @@ -305,13 +281,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "watsonx/ibm/granite-3-8b-instruct": { "max_tokens": 8192, @@ -349,13 +319,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "gpt-4o-search-preview": { "max_tokens": 16384, @@ -527,13 +491,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview-2025-03-11": { "max_tokens": 16384, @@ -553,13 +511,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview": { "max_tokens": 16384, @@ -954,13 +906,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "gpt-4o-2024-11-20": { "max_tokens": 16384, @@ -4217,7 +4163,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision-1212": { "max_tokens": 32768, @@ -4230,7 +4177,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision-latest": { "max_tokens": 32768, @@ -4243,7 +4191,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision": { "max_tokens": 32768, @@ -4256,7 +4205,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-3": { "max_tokens": 131072, @@ -4269,7 +4219,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-beta": { "max_tokens": 131072, @@ -4282,7 +4233,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-fast-beta": { "max_tokens": 131072, @@ -4295,7 +4247,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-fast-latest": { "max_tokens": 131072, @@ -4308,7 +4261,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-beta": { "max_tokens": 131072, @@ -4322,7 +4276,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-fast-beta": { "max_tokens": 131072, @@ -4336,7 +4291,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-fast-latest": { "max_tokens": 131072, @@ -4350,7 +4306,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-vision-beta": { "max_tokens": 8192, @@ -4363,7 +4320,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-1212": { "max_tokens": 131072, @@ -4374,7 +4332,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2": { "max_tokens": 131072, @@ -4385,7 +4344,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-latest": { "max_tokens": 131072, @@ -4396,7 +4356,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "deepseek/deepseek-coder": { "max_tokens": 4096, @@ -6112,7 +6073,8 @@ "text" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-pro-exp-02-05": { "max_tokens": 8192, @@ -6152,7 +6114,8 @@ "text" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-exp": { "max_tokens": 8192, @@ -6197,7 +6160,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-001": { "max_tokens": 8192, @@ -6232,7 +6196,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "deprecation_date": "2026-02-05", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-thinking-exp": { "max_tokens": 8192, @@ -6277,7 +6242,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-thinking-exp-01-21": { "max_tokens": 65536, @@ -6322,7 +6288,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini/gemini-2.5-pro-exp-03-25": { "max_tokens": 65535, @@ -6363,7 +6330,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-tts": { "max_tokens": 65535, @@ -6400,7 +6368,8 @@ "supported_output_modalities": [ "audio" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-05-20": { "max_tokens": 65535, @@ -6440,7 +6409,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-04-17": { "max_tokens": 65535, @@ -6480,7 +6450,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini-2.5-flash-preview-05-20": { "max_tokens": 65535, @@ -6520,7 +6491,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-flash-preview-04-17": { "max_tokens": 65535, @@ -6560,7 +6532,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash": { "max_tokens": 8192, @@ -6595,7 +6568,8 @@ ], "supports_tool_choice": true, "source": "https://ai.google.dev/pricing#2_0flash", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-lite": { "max_input_tokens": 1048576, @@ -6627,7 +6601,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-lite-001": { "max_input_tokens": 1048576, @@ -6660,7 +6635,8 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, "deprecation_date": "2026-02-25", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-05-06": { "max_tokens": 65535, @@ -6701,7 +6677,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-03-25": { "max_tokens": 65535, @@ -6742,7 +6719,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-preview-image-generation": { "max_tokens": 8192, @@ -6777,7 +6755,8 @@ ], "supports_tool_choice": true, "source": "https://ai.google.dev/pricing#2_0flash", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-tts": { "max_tokens": 65535, @@ -6809,7 +6788,8 @@ "audio" ], "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini/gemini-2.0-pro-exp-02-05": { "max_tokens": 8192, @@ -6847,7 +6827,8 @@ "supports_pdf_input": true, "supports_response_schema": true, "supports_tool_choice": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_web_search": true }, "gemini/gemini-2.0-flash-preview-image-generation": { "max_tokens": 8192, @@ -6883,7 +6864,8 @@ "image" ], "supports_tool_choice": true, - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.0-flash": { "max_tokens": 8192, @@ -6919,7 +6901,8 @@ "image" ], "supports_tool_choice": true, - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.0-flash-lite": { "max_input_tokens": 1048576, @@ -6952,7 +6935,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supports_web_search": true }, "gemini/gemini-2.0-flash-001": { "max_tokens": 8192, @@ -6987,7 +6971,8 @@ "text", "image" ], - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-tts": { "max_tokens": 65535, @@ -7020,7 +7005,8 @@ "supported_output_modalities": [ "audio" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-05-06": { "max_tokens": 65535, @@ -7056,7 +7042,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-03-25": { "max_tokens": 65535, @@ -7092,7 +7079,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.0-flash-exp": { "max_tokens": 8192, @@ -7138,7 +7126,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemini-2.0-flash-lite-preview-02-05": { "max_tokens": 8192, @@ -7172,7 +7161,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash-lite" + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash-lite", + "supports_web_search": true }, "gemini/gemini-2.0-flash-thinking-exp": { "max_tokens": 8192, @@ -7218,7 +7208,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemini-2.0-flash-thinking-exp-01-21": { "max_tokens": 8192, @@ -7264,7 +7255,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemma-3-27b-it": { "max_tokens": 8192, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index ded872e0ce..dd0c35ea84 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,5 +3,16 @@ model_list: litellm_params: model: anthropic/* +guardrails: + - guardrail_name: "bedrock-pre-guard" + litellm_params: + guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" + mode: "post_call" + guardrailIdentifier: wf0hkdb5x07f # your guardrail ID on bedrock + guardrailVersion: "DRAFT" # your guardrail version on bedrock + default_on: true + + general_settings: store_prompts_in_spend_logs: true + diff --git a/litellm/utils.py b/litellm/utils.py index 6d42e49607..4bb4cb5e83 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1885,6 +1885,22 @@ def supports_tool_choice(model: str, custom_llm_provider: Optional[str] = None) ) +def _supports_provider_info_factory( + model: str, custom_llm_provider: Optional[str], key: str +) -> Optional[Literal[True]]: + """ + Check if the given model supports a provider specific model info and return a boolean value. + """ + + provider_info = get_provider_info( + model=model, custom_llm_provider=custom_llm_provider + ) + + if provider_info is not None and provider_info.get(key, False) is True: + return True + return None + + def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) -> bool: """ Check if the given model supports function calling and return a boolean value. @@ -1910,18 +1926,25 @@ def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) if model_info.get(key, False) is True: return True + elif model_info.get(key) is None: # don't check if 'False' explicitly set + supported_by_provider = _supports_provider_info_factory( + model, custom_llm_provider, key + ) + if supported_by_provider is not None: + return supported_by_provider + return False except Exception as e: verbose_logger.debug( f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {str(e)}" ) - provider_info = get_provider_info( - model=model, custom_llm_provider=custom_llm_provider + supported_by_provider = _supports_provider_info_factory( + model, custom_llm_provider, key ) + if supported_by_provider is not None: + return supported_by_provider - if provider_info is not None and provider_info.get(key, False) is True: - return True return False @@ -4504,29 +4527,29 @@ def _get_model_info_helper( # noqa: PLR0915 supports_response_schema=_model_info.get( "supports_response_schema", None ), - supports_vision=_model_info.get("supports_vision", False), + supports_vision=_model_info.get("supports_vision", None), supports_function_calling=_model_info.get( - "supports_function_calling", False + "supports_function_calling", None ), - supports_tool_choice=_model_info.get("supports_tool_choice", False), + supports_tool_choice=_model_info.get("supports_tool_choice", None), supports_assistant_prefill=_model_info.get( - "supports_assistant_prefill", False + "supports_assistant_prefill", None ), supports_prompt_caching=_model_info.get( - "supports_prompt_caching", False + "supports_prompt_caching", None ), - supports_audio_input=_model_info.get("supports_audio_input", False), - supports_audio_output=_model_info.get("supports_audio_output", False), - supports_pdf_input=_model_info.get("supports_pdf_input", False), + supports_audio_input=_model_info.get("supports_audio_input", None), + supports_audio_output=_model_info.get("supports_audio_output", None), + supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get( - "supports_embedding_image_input", False + "supports_embedding_image_input", None ), supports_native_streaming=_model_info.get( "supports_native_streaming", None ), - supports_web_search=_model_info.get("supports_web_search", False), - supports_reasoning=_model_info.get("supports_reasoning", False), - supports_computer_use=_model_info.get("supports_computer_use", False), + supports_web_search=_model_info.get("supports_web_search", None), + supports_reasoning=_model_info.get("supports_reasoning", None), + supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get( "search_context_cost_per_query", None ), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 988000c640..4432be88b6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -96,13 +96,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_native_streaming": true }, "gpt-4.1-2025-04-14": { "max_tokens": 32768, @@ -135,13 +129,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_native_streaming": true }, "gpt-4.1-mini": { "max_tokens": 32768, @@ -174,13 +162,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_native_streaming": true }, "gpt-4.1-mini-2025-04-14": { "max_tokens": 32768, @@ -213,13 +195,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_native_streaming": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_native_streaming": true }, "gpt-4.1-nano": { "max_tokens": 32768, @@ -305,13 +281,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "watsonx/ibm/granite-3-8b-instruct": { "max_tokens": 8192, @@ -349,13 +319,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "gpt-4o-search-preview": { "max_tokens": 16384, @@ -527,13 +491,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview-2025-03-11": { "max_tokens": 16384, @@ -553,13 +511,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.03 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview": { "max_tokens": 16384, @@ -954,13 +906,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.05 - } + "supports_tool_choice": true }, "gpt-4o-2024-11-20": { "max_tokens": 16384, @@ -4217,7 +4163,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision-1212": { "max_tokens": 32768, @@ -4230,7 +4177,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision-latest": { "max_tokens": 32768, @@ -4243,7 +4191,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-vision": { "max_tokens": 32768, @@ -4256,7 +4205,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-3": { "max_tokens": 131072, @@ -4269,7 +4219,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-beta": { "max_tokens": 131072, @@ -4282,7 +4233,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-fast-beta": { "max_tokens": 131072, @@ -4295,7 +4247,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-fast-latest": { "max_tokens": 131072, @@ -4308,7 +4261,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-beta": { "max_tokens": 131072, @@ -4322,7 +4276,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-fast-beta": { "max_tokens": 131072, @@ -4336,7 +4291,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-3-mini-fast-latest": { "max_tokens": 131072, @@ -4350,7 +4306,8 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": false, - "source": "https://x.ai/api#pricing" + "source": "https://x.ai/api#pricing", + "supports_web_search": true }, "xai/grok-vision-beta": { "max_tokens": 8192, @@ -4363,7 +4320,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-1212": { "max_tokens": 131072, @@ -4374,7 +4332,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2": { "max_tokens": 131072, @@ -4385,7 +4344,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "xai/grok-2-latest": { "max_tokens": 131072, @@ -4396,7 +4356,8 @@ "litellm_provider": "xai", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "deepseek/deepseek-coder": { "max_tokens": 4096, @@ -6112,7 +6073,8 @@ "text" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-pro-exp-02-05": { "max_tokens": 8192, @@ -6152,7 +6114,8 @@ "text" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-exp": { "max_tokens": 8192, @@ -6197,7 +6160,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-001": { "max_tokens": 8192, @@ -6232,7 +6196,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "deprecation_date": "2026-02-05", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-thinking-exp": { "max_tokens": 8192, @@ -6277,7 +6242,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-thinking-exp-01-21": { "max_tokens": 65536, @@ -6322,7 +6288,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini/gemini-2.5-pro-exp-03-25": { "max_tokens": 65535, @@ -6363,7 +6330,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-tts": { "max_tokens": 65535, @@ -6400,7 +6368,8 @@ "supported_output_modalities": [ "audio" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-05-20": { "max_tokens": 65535, @@ -6440,7 +6409,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini/gemini-2.5-flash-preview-04-17": { "max_tokens": 65535, @@ -6480,7 +6450,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview" + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supports_web_search": true }, "gemini-2.5-flash-preview-05-20": { "max_tokens": 65535, @@ -6520,7 +6491,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-flash-preview-04-17": { "max_tokens": 65535, @@ -6560,7 +6532,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash": { "max_tokens": 8192, @@ -6595,7 +6568,8 @@ ], "supports_tool_choice": true, "source": "https://ai.google.dev/pricing#2_0flash", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-lite": { "max_input_tokens": 1048576, @@ -6627,7 +6601,8 @@ ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-lite-001": { "max_input_tokens": 1048576, @@ -6660,7 +6635,8 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", "supports_tool_choice": true, "deprecation_date": "2026-02-25", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-05-06": { "max_tokens": 65535, @@ -6701,7 +6677,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-03-25": { "max_tokens": 65535, @@ -6742,7 +6719,8 @@ "text" ], "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.0-flash-preview-image-generation": { "max_tokens": 8192, @@ -6777,7 +6755,8 @@ ], "supports_tool_choice": true, "source": "https://ai.google.dev/pricing#2_0flash", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini-2.5-pro-preview-tts": { "max_tokens": 65535, @@ -6809,7 +6788,8 @@ "audio" ], "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supports_web_search": true }, "gemini/gemini-2.0-pro-exp-02-05": { "max_tokens": 8192, @@ -6847,7 +6827,8 @@ "supports_pdf_input": true, "supports_response_schema": true, "supports_tool_choice": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_web_search": true }, "gemini/gemini-2.0-flash-preview-image-generation": { "max_tokens": 8192, @@ -6883,7 +6864,8 @@ "image" ], "supports_tool_choice": true, - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.0-flash": { "max_tokens": 8192, @@ -6919,7 +6901,8 @@ "image" ], "supports_tool_choice": true, - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.0-flash-lite": { "max_input_tokens": 1048576, @@ -6952,7 +6935,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supports_web_search": true }, "gemini/gemini-2.0-flash-001": { "max_tokens": 8192, @@ -6987,7 +6971,8 @@ "text", "image" ], - "source": "https://ai.google.dev/pricing#2_0flash" + "source": "https://ai.google.dev/pricing#2_0flash", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-tts": { "max_tokens": 65535, @@ -7020,7 +7005,8 @@ "supported_output_modalities": [ "audio" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-05-06": { "max_tokens": 65535, @@ -7056,7 +7042,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.5-pro-preview-03-25": { "max_tokens": 65535, @@ -7092,7 +7079,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview" + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview", + "supports_web_search": true }, "gemini/gemini-2.0-flash-exp": { "max_tokens": 8192, @@ -7138,7 +7126,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemini-2.0-flash-lite-preview-02-05": { "max_tokens": 8192, @@ -7172,7 +7161,8 @@ "supported_output_modalities": [ "text" ], - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash-lite" + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash-lite", + "supports_web_search": true }, "gemini/gemini-2.0-flash-thinking-exp": { "max_tokens": 8192, @@ -7218,7 +7208,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemini-2.0-flash-thinking-exp-01-21": { "max_tokens": 8192, @@ -7264,7 +7255,8 @@ "image" ], "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "gemini/gemma-3-27b-it": { "max_tokens": 8192, diff --git a/test_script.py b/test_script.py new file mode 100644 index 0000000000..8458b95de3 --- /dev/null +++ b/test_script.py @@ -0,0 +1,13 @@ +import os +import json + +gemini_model_cost_map = json.load(open("model_prices_and_context_window.json")) + +for model, model_info in gemini_model_cost_map.items(): + if model_info.get("litellm_provider") == "gemini" and model_info.get("mode") == "chat" and "gemini-2" in model: + """ + Update all gemini chat models to support web search + """ + model_info["supports_web_search"] = True + +json.dump(gemini_model_cost_map, open("model_prices_and_context_window.json", "w"), indent=4) \ No newline at end of file diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index c2f0d7a1a9..2bfde87fbd 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -21,7 +21,8 @@ IGNORE_FUNCTIONS = [ "set_schema_property_ordering", # testing added for infinite recursion "process_items", # testing added for infinite recursion + max depth set. "_can_object_call_model", # max depth set. - "encode_unserializable_types" # max depth set. + "encode_unserializable_types", # max depth set. + "filter_value_from_dict", # max depth set. ] diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 46e409f54c..998c27a93d 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -7,8 +7,7 @@ import litellm from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache -from unittest.mock import MagicMock -from unittest.mock import AsyncMock, patch +from unittest.mock import MagicMock, AsyncMock, patch @pytest.mark.asyncio async def test_bedrock_guardrails(): @@ -334,4 +333,5 @@ async def test_bedrock_guardrail_aws_param_persistence(): assert kwargs["aws_access_key_id"] == "test-access-key" assert kwargs["aws_secret_access_key"] == "test-secret-key" assert kwargs["aws_region_name"] == "us-east-1" + \ No newline at end of file diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index b8007c2d34..26788229c3 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -56,7 +56,8 @@ def _usage_format_tests(usage: litellm.Usage): print(f"usage={usage}") assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - assert usage.prompt_tokens > usage.prompt_tokens_details.cached_tokens + if usage.prompt_tokens_details is not None: + assert usage.prompt_tokens > usage.prompt_tokens_details.cached_tokens class BaseLLMChatTest(ABC): @@ -184,6 +185,30 @@ class BaseLLMChatTest(ABC): self.completion_function(**base_completion_call_args, messages=messages) + + def test_web_search(self): + from litellm.utils import supports_web_search + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm._turn_on_debug() + + base_completion_call_args = self.get_base_completion_call_args() + + if not supports_web_search(base_completion_call_args["model"], None): + pytest.skip("Model does not support web search") + + response = self.completion_function( + **base_completion_call_args, + messages=[{"role": "user", "content": "What's the weather like in Boston today?"}], + web_search_options={}, + max_tokens=100, + ) + + assert response is not None + + print(f"response={response}") + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_pdf_handling(self, pdf_messages, sync_mode): @@ -847,9 +872,10 @@ class BaseLLMChatTest(ABC): _usage_format_tests(response.usage) assert "prompt_tokens_details" in response.usage - assert ( - response.usage.prompt_tokens_details.cached_tokens > 0 - ), f"cached_tokens={response.usage.prompt_tokens_details.cached_tokens} should be greater than 0. Got usage={response.usage}" + if response.usage.prompt_tokens_details is not None: + assert ( + response.usage.prompt_tokens_details.cached_tokens > 0 + ), f"cached_tokens={response.usage.prompt_tokens_details.cached_tokens} should be greater than 0. Got usage={response.usage}" except litellm.InternalServerError: pass @@ -978,7 +1004,7 @@ class BaseLLMChatTest(ABC): assert isinstance( response.choices[0].message.tool_calls[0].function.arguments, str ) - assert response.choices[0].finish_reason == "tool_calls" + assert response.choices[0].finish_reason == "tool_calls", f"finish_reason: {response.choices[0].finish_reason}, expected: tool_calls" messages.append( response.choices[0].message.model_dump() ) # Add assistant tool invokes diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 8678414ddb..de38101f13 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -465,7 +465,7 @@ from litellm import completion class TestAnthropicCompletion(BaseLLMChatTest, BaseAnthropicChatTest): def get_base_completion_call_args(self) -> dict: - return {"model": "anthropic/claude-3-5-sonnet-20240620"} + return {"model": "anthropic/claude-3-5-sonnet-latest"} def get_base_completion_call_args_with_thinking(self) -> dict: return { diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index 8f9619a2b7..648ae2b15b 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -18,7 +18,7 @@ from litellm import Choices, Message, ModelResponse, EmbeddingResponse, Usage from litellm import completion from unittest.mock import patch from litellm.llms.xai.chat.transformation import XAIChatConfig, XAI_API_BASE -from base_llm_unit_tests import BaseReasoningLLMTests +from base_llm_unit_tests import BaseReasoningLLMTests, BaseLLMChatTest def test_xai_chat_config_get_openai_compatible_provider_info(): @@ -169,3 +169,12 @@ class TestXAIReasoningEffort(BaseReasoningLLMTests): "model": "xai/grok-3-mini-beta", "messages": [{"role": "user", "content": "Hello"}], } + +class TestXAIChat(BaseLLMChatTest): + def get_base_completion_call_args(self): + return { + "model": "xai/grok-3-mini-beta", + } + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex.py b/tests/test_litellm/llms/vertex_ai/test_vertex.py index 1b9de4d0fd..d4679b5ceb 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex.py @@ -1432,7 +1432,7 @@ def test_vertex_parallel_tool_calls_false_multiple_tools_error(): parallel_tool_calls=False, ) assert ( - "`parallel_tool_calls=False` is not supported when multiple tools are provided" + "`parallel_tool_calls=False` is not supported by Gemini when multiple tools are" in str(excinfo.value) ) @@ -1445,7 +1445,7 @@ def test_vertex_parallel_tool_calls_false_multiple_tools_error(): parallel_tool_calls=False, ) assert ( - "`parallel_tool_calls=False` is not supported when multiple tools are provided" + "`parallel_tool_calls=False` is not supported by Gemini when multiple tools are" in str(excinfo.value) ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 0d57da0ed2..1ca834b65c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -759,8 +759,8 @@ def test_get_model_info_shows_supports_computer_use(): info_gpt = litellm.get_model_info(model_known_not_to_support_computer_use) print(f"Info for {model_known_not_to_support_computer_use}: {info_gpt}") assert ( - info_gpt.get("supports_computer_use") is False - ) # Expecting False due to the default in ModelInfoBase + info_gpt.get("supports_computer_use") is None + ) # Expecting None due to the default in ModelInfoBase @pytest.mark.parametrize(