From 4683b8ea7031995883d227d4e260034c71de7c47 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 10:24:08 -0700 Subject: [PATCH 01/16] add search_context_cost_per_1k_calls --- ...odel_prices_and_context_window_backup.json | 45 +++++++++++++++++-- model_prices_and_context_window.json | 45 +++++++++++++++++-- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c5e0329013..8733730946 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -74,7 +74,36 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } + }, + "gpt-4o-search-preview": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "input_cost_per_token_batches": 0.00000125, + "output_cost_per_token_batches": 0.00000500, + "cache_read_input_token_cost": 0.00000125, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "gpt-4.5-preview": { "max_tokens": 16384, @@ -199,7 +228,12 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "gpt-4o-mini-2024-07-18": { "max_tokens": 16384, @@ -218,7 +252,12 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "o1-pro": { "max_tokens": 100000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c5e0329013..8733730946 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -74,7 +74,36 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } + }, + "gpt-4o-search-preview": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "input_cost_per_token_batches": 0.00000125, + "output_cost_per_token_batches": 0.00000500, + "cache_read_input_token_cost": 0.00000125, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "gpt-4.5-preview": { "max_tokens": 16384, @@ -199,7 +228,12 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "gpt-4o-mini-2024-07-18": { "max_tokens": 16384, @@ -218,7 +252,12 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } }, "o1-pro": { "max_tokens": 100000, From dc5bf10009b7c95e32cac19f612391d43e1e058e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 10:43:35 -0700 Subject: [PATCH 02/16] feat - add openai web search --- .../convert_dict_to_response.py | 1 + litellm/types/llms/openai.py | 22 +++++++++++++++++++ litellm/types/utils.py | 7 ++++++ 3 files changed, 30 insertions(+) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index ebb1032a19..d33af2a477 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -494,6 +494,7 @@ def convert_to_model_response_object( # noqa: PLR0915 provider_specific_fields=provider_specific_fields, reasoning_content=reasoning_content, thinking_blocks=thinking_blocks, + annotations=choice["message"].get("annotations", None), ) finish_reason = choice.get("finish_reason", None) if finish_reason is None: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4b0be9d5fe..e58f573227 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -382,6 +382,28 @@ class ChatCompletionThinkingBlock(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] +class ChatCompletionAnnotationURLCitation(TypedDict, total=False): + end_index: int + """The index of the last character of the URL citation in the message.""" + + start_index: int + """The index of the first character of the URL citation in the message.""" + + title: str + """The title of the web resource.""" + + url: str + """The URL of the web resource.""" + + +class ChatCompletionAnnotation(TypedDict, total=False): + type: Literal["url_citation"] + """The type of the URL citation. Always `url_citation`.""" + + url_citation: ChatCompletionAnnotationURLCitation + """A URL citation when using web search.""" + + class OpenAIChatCompletionTextObject(TypedDict): type: Literal["text"] text: str diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a665428561..58c78dfa29 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union from aiohttp import FormData from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import FileTypes # type: ignore +from openai.types.chat.chat_completion import ChatCompletion from openai.types.completion_usage import ( CompletionTokensDetails, CompletionUsage, @@ -27,6 +28,7 @@ from ..litellm_core_utils.core_helpers import map_finish_reason from .guardrails import GuardrailEventHooks from .llms.openai import ( Batch, + ChatCompletionAnnotation, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionUsageBlock, @@ -527,6 +529,7 @@ class Message(OpenAIObject): provider_specific_fields: Optional[Dict[str, Any]] = Field( default=None, exclude=True ) + annotations: Optional[List[ChatCompletionAnnotation]] = None def __init__( self, @@ -538,6 +541,7 @@ class Message(OpenAIObject): provider_specific_fields: Optional[Dict[str, Any]] = None, reasoning_content: Optional[str] = None, thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None, + annotations: Optional[List[ChatCompletionAnnotation]] = None, **params, ): init_values: Dict[str, Any] = { @@ -566,6 +570,9 @@ class Message(OpenAIObject): if thinking_blocks is not None: init_values["thinking_blocks"] = thinking_blocks + if annotations is not None: + init_values["annotations"] = annotations + if reasoning_content is not None: init_values["reasoning_content"] = reasoning_content From 3764aa17298698e499d8a18b507eb6b878be9ce6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 10:44:04 -0700 Subject: [PATCH 03/16] test open ai web search --- tests/llm_translation/test_openai.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 172c946636..456190469a 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -391,3 +391,17 @@ def test_openai_chat_completion_streaming_handler_reasoning_content(): ) assert response.choices[0].delta.reasoning_content == "." + + +def test_openai_web_search(): + # litellm._turn_on_debug() + response = litellm.completion( + model="openai/gpt-4o-search-preview", + messages=[ + { + "role": "user", + "content": "What was a positive news story from today?", + } + ], + ) + print("litellm response: ", response.model_dump_json(indent=4)) From 3dbbc89fd274075fdaf5e1b85be130a9d00fc2dc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 10:53:47 -0700 Subject: [PATCH 04/16] test_openai_web_search --- tests/llm_translation/test_openai.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 456190469a..97af8d93e4 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -17,6 +17,10 @@ import litellm from litellm import Choices, Message, ModelResponse from base_llm_unit_tests import BaseLLMChatTest import asyncio +from litellm.types.llms.openai import ( + ChatCompletionAnnotation, + ChatCompletionAnnotationURLCitation, +) def test_openai_prediction_param(): @@ -393,8 +397,28 @@ def test_openai_chat_completion_streaming_handler_reasoning_content(): assert response.choices[0].delta.reasoning_content == "." +def validate_response_url_citation(url_citation: ChatCompletionAnnotationURLCitation): + assert "end_index" in url_citation + assert "start_index" in url_citation + assert "url" in url_citation + + +def validate_model_response_contains_web_search_annotations(response: ModelResponse): + """validates litellm response contains web search annotations""" + message = response.choices[0].message + annotations: ChatCompletionAnnotation = message.annotations + print("annotations: ", annotations) + assert annotations is not None + assert isinstance(annotations, list) + for annotation in annotations: + assert annotation["type"] == "url_citation" + url_citation: ChatCompletionAnnotationURLCitation = annotation["url_citation"] + validate_response_url_citation(url_citation) + + def test_openai_web_search(): - # litellm._turn_on_debug() + """Makes a simple web search request and validates the response contains web search annotations and all expected fields are present""" + litellm._turn_on_debug() response = litellm.completion( model="openai/gpt-4o-search-preview", messages=[ @@ -405,3 +429,4 @@ def test_openai_web_search(): ], ) print("litellm response: ", response.model_dump_json(indent=4)) + validate_model_response_contains_web_search_annotations(response) From 5c59a6a58f0554da220b00024f439edfe4810a1d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 11:36:34 -0700 Subject: [PATCH 05/16] test_openai_web_search_streaming --- tests/llm_translation/test_openai.py | 36 ++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 97af8d93e4..633ff76467 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -3,6 +3,7 @@ import os import sys from datetime import datetime from unittest.mock import AsyncMock, patch +from typing import Optional sys.path.insert( 0, os.path.abspath("../..") @@ -403,10 +404,8 @@ def validate_response_url_citation(url_citation: ChatCompletionAnnotationURLCita assert "url" in url_citation -def validate_model_response_contains_web_search_annotations(response: ModelResponse): +def validate_web_search_annotations(annotations: ChatCompletionAnnotation): """validates litellm response contains web search annotations""" - message = response.choices[0].message - annotations: ChatCompletionAnnotation = message.annotations print("annotations: ", annotations) assert annotations is not None assert isinstance(annotations, list) @@ -429,4 +428,33 @@ def test_openai_web_search(): ], ) print("litellm response: ", response.model_dump_json(indent=4)) - validate_model_response_contains_web_search_annotations(response) + message = response.choices[0].message + annotations: ChatCompletionAnnotation = message.annotations + validate_web_search_annotations(annotations) + + +def test_openai_web_search_streaming(): + """Makes a simple web search request and validates the response contains web search annotations and all expected fields are present""" + # litellm._turn_on_debug() + test_openai_web_search: Optional[ChatCompletionAnnotation] = None + response = litellm.completion( + model="openai/gpt-4o-search-preview", + messages=[ + { + "role": "user", + "content": "What was a positive news story from today?", + } + ], + stream=True, + ) + for chunk in response: + print("litellm response chunk: ", chunk) + if ( + hasattr(chunk.choices[0].delta, "annotations") + and chunk.choices[0].delta.annotations is not None + ): + test_openai_web_search = chunk.choices[0].delta.annotations + + # Assert this request has at-least one web search annotation + assert test_openai_web_search is not None + validate_web_search_annotations(test_openai_web_search) From 44f4c623e2a94024c63f2d212c3683ead14727d4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 11:38:30 -0700 Subject: [PATCH 06/16] Add annotations to the delta --- litellm/types/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 58c78dfa29..8821d2c80b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -630,6 +630,7 @@ class Delta(OpenAIObject): audio: Optional[ChatCompletionAudioResponse] = None, reasoning_content: Optional[str] = None, thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None, + annotations: Optional[List[ChatCompletionAnnotation]] = None, **params, ): super(Delta, self).__init__(**params) @@ -640,6 +641,7 @@ class Delta(OpenAIObject): self.function_call: Optional[Union[FunctionCall, Any]] = None self.tool_calls: Optional[List[Union[ChatCompletionDeltaToolCall, Any]]] = None self.audio: Optional[ChatCompletionAudioResponse] = None + self.annotations: Optional[List[ChatCompletionAnnotation]] = None if reasoning_content is not None: self.reasoning_content = reasoning_content @@ -653,6 +655,12 @@ class Delta(OpenAIObject): # ensure default response matches OpenAI spec del self.thinking_blocks + # Add annotations to the delta, ensure they are only on Delta if they exist (Match OpenAI spec) + if annotations is not None: + self.annotations = annotations + else: + del self.annotations + if function_call is not None and isinstance(function_call, dict): self.function_call = FunctionCall(**function_call) else: From 792a2d61150b10ac8210e6376b468eeb5f60ae59 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 11:41:53 -0700 Subject: [PATCH 07/16] test_is_chunk_non_empty_with_annotations --- .../litellm_core_utils/streaming_handler.py | 5 ++- .../test_streaming_handler.py | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 56e64d1859..a11e5af12b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -799,6 +799,10 @@ class CustomStreamWrapper: "provider_specific_fields" in response_obj and response_obj["provider_specific_fields"] is not None ) + or ( + "annotations" in model_response.choices[0].delta + and model_response.choices[0].delta.annotations is not None + ) ): return True else: @@ -939,7 +943,6 @@ class CustomStreamWrapper: and model_response.choices[0].delta.audio is not None ): return model_response - else: if hasattr(model_response, "usage"): self.chunks.append(model_response) diff --git a/tests/litellm/litellm_core_utils/test_streaming_handler.py b/tests/litellm/litellm_core_utils/test_streaming_handler.py index 75c4fc1035..988d533670 100644 --- a/tests/litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/litellm/litellm_core_utils/test_streaming_handler.py @@ -136,6 +136,40 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp ) +def test_is_chunk_non_empty_with_annotations( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """Unit test if non-empty when annotations are present""" + chunk = { + "id": "e89b6501-8ac2-464c-9550-7cd3daf94350", + "object": "chat.completion.chunk", + "created": 1741037890, + "model": "deepseek-reasoner", + "system_fingerprint": "fp_5417b77867_prod0225", + "choices": [ + { + "index": 0, + "delta": { + "content": None, + "annotations": [ + {"type": "url_citation", "url": "https://www.google.com"} + ], + }, + "logprobs": None, + "finish_reason": None, + } + ], + } + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj=MagicMock(), + model_response=ModelResponseStream(**chunk), + response_obj=MagicMock(), + ) + is True + ) + + def test_optional_combine_thinking_block_in_choices( initialized_custom_stream_wrapper: CustomStreamWrapper, ): From 1bdb94a3148698776f2acea65c976d8eac49b33a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 12:56:21 -0700 Subject: [PATCH 08/16] add search_context_cost_per_1k_calls to model cost map spec --- tests/local_testing/test_get_model_info.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index d71f3f7c24..f885a61e78 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -518,6 +518,15 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): ], }, }, + "search_context_cost_per_1k_calls": { + "type": "object", + "properties": { + "low_context": {"type": "number"}, + "medium_context": {"type": "number"}, + "high_context": {"type": "number"}, + }, + "additionalProperties": False, + }, "supported_modalities": { "type": "array", "items": { From d2e36d8bedacb5047f4c9b22a3f46b5a8a05ff16 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 12:57:00 -0700 Subject: [PATCH 09/16] search_context_cost_per_1k_calls --- litellm/model_prices_and_context_window_backup.json | 5 +++++ model_prices_and_context_window.json | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8733730946..b666c84446 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15,6 +15,11 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, + "search_context_cost_per_1k_calls": { + "low_context": 0.0000, + "medium_context": 0.0000, + "high_context": 0.0000 + }, "deprecation_date": "date when the model becomes deprecated in the format YYYY-MM-DD" }, "omni-moderation-latest": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8733730946..b666c84446 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15,6 +15,11 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, + "search_context_cost_per_1k_calls": { + "low_context": 0.0000, + "medium_context": 0.0000, + "high_context": 0.0000 + }, "deprecation_date": "date when the model becomes deprecated in the format YYYY-MM-DD" }, "omni-moderation-latest": { From baa2bdb762e9e72eb414492ab4b7341aa5907cf4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 12:59:50 -0700 Subject: [PATCH 10/16] add supports_web_search --- ...odel_prices_and_context_window_backup.json | 29 +++++++++++++++++++ model_prices_and_context_window.json | 29 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b666c84446..0c3b82e6fa 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15,6 +15,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 0.0000, "medium_context": 0.0000, @@ -80,6 +81,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, @@ -104,6 +106,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, @@ -234,6 +237,32 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } + }, + "gpt-4o-mini-search-preview": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000015, + "output_cost_per_token": 0.00000060, + "input_cost_per_token_batches": 0.000000075, + "output_cost_per_token_batches": 0.00000030, + "cache_read_input_token_cost": 0.000000075, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b666c84446..0c3b82e6fa 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15,6 +15,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 0.0000, "medium_context": 0.0000, @@ -80,6 +81,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, @@ -104,6 +106,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, @@ -234,6 +237,32 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_web_search": true, + "search_context_cost_per_1k_calls": { + "low_context": 30.00, + "medium_context": 35.00, + "high_context": 50.00 + } + }, + "gpt-4o-mini-search-preview": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000015, + "output_cost_per_token": 0.00000060, + "input_cost_per_token_batches": 0.000000075, + "output_cost_per_token_batches": 0.00000030, + "cache_read_input_token_cost": 0.000000075, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_web_search": true, "search_context_cost_per_1k_calls": { "low_context": 30.00, "medium_context": 35.00, From c4cbfd57162c53a2345b31ba40bb71f61d14f3e5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:01:41 -0700 Subject: [PATCH 11/16] supports_web_search --- litellm/utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 52dbccb0c8..03e69acf4e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1975,6 +1975,27 @@ def supports_system_messages(model: str, custom_llm_provider: Optional[str]) -> ) +def supports_web_search(model: str, custom_llm_provider: Optional[str]) -> bool: + """ + Check if the given model supports web search and return a boolean value. + + Parameters: + model (str): The model name to be checked. + custom_llm_provider (str): The provider to be checked. + + Returns: + bool: True if the model supports web search, False otherwise. + + Raises: + Exception: If the given model is not found in model_prices_and_context_window.json. + """ + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_web_search", + ) + + def supports_native_streaming(model: str, custom_llm_provider: Optional[str]) -> bool: """ Check if the given model supports native streaming and return a boolean value. From df6901f39e79999d0c93e899acdfb42304b401af Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:04:24 -0700 Subject: [PATCH 12/16] search_context_cost_per_1k_calls --- ...odel_prices_and_context_window_backup.json | 36 +++++++++---------- model_prices_and_context_window.json | 36 +++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0c3b82e6fa..33d01523e1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17,9 +17,9 @@ "supports_system_messages": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 0.0000, - "medium_context": 0.0000, - "high_context": 0.0000 + "search_context_size_low": 0.0000, + "search_context_size_medium": 0.0000, + "search_context_size_high": 0.0000 }, "deprecation_date": "date when the model becomes deprecated in the format YYYY-MM-DD" }, @@ -83,9 +83,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "gpt-4o-search-preview": { @@ -108,9 +108,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "gpt-4.5-preview": { @@ -239,9 +239,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 25.00, + "search_context_size_medium": 27.50, + "search_context_size_high": 30.00 } }, "gpt-4o-mini-search-preview": { @@ -264,9 +264,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 25.00, + "search_context_size_medium": 27.50, + "search_context_size_high": 30.00 } }, "gpt-4o-mini-2024-07-18": { @@ -288,9 +288,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "o1-pro": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0c3b82e6fa..33d01523e1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17,9 +17,9 @@ "supports_system_messages": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 0.0000, - "medium_context": 0.0000, - "high_context": 0.0000 + "search_context_size_low": 0.0000, + "search_context_size_medium": 0.0000, + "search_context_size_high": 0.0000 }, "deprecation_date": "date when the model becomes deprecated in the format YYYY-MM-DD" }, @@ -83,9 +83,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "gpt-4o-search-preview": { @@ -108,9 +108,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "gpt-4.5-preview": { @@ -239,9 +239,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 25.00, + "search_context_size_medium": 27.50, + "search_context_size_high": 30.00 } }, "gpt-4o-mini-search-preview": { @@ -264,9 +264,9 @@ "supports_tool_choice": true, "supports_web_search": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 25.00, + "search_context_size_medium": 27.50, + "search_context_size_high": 30.00 } }, "gpt-4o-mini-2024-07-18": { @@ -288,9 +288,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "search_context_cost_per_1k_calls": { - "low_context": 30.00, - "medium_context": 35.00, - "high_context": 50.00 + "search_context_size_low": 30.00, + "search_context_size_medium": 35.00, + "search_context_size_high": 50.00 } }, "o1-pro": { From 373d59720b8c7b3a9b6b5a720c33ad1ba679a648 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:07:17 -0700 Subject: [PATCH 13/16] search_context_cost_per_query --- ...odel_prices_and_context_window_backup.json | 36 +++++++++---------- model_prices_and_context_window.json | 36 +++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 33d01523e1..6992e1afcf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16,7 +16,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { + "search_context_cost_per_query": { "search_context_size_low": 0.0000, "search_context_size_medium": 0.0000, "search_context_size_high": 0.0000 @@ -82,10 +82,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 30.00, - "search_context_size_medium": 35.00, - "search_context_size_high": 50.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.030, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.050 } }, "gpt-4o-search-preview": { @@ -107,10 +107,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 30.00, - "search_context_size_medium": 35.00, - "search_context_size_high": 50.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.030, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.050 } }, "gpt-4.5-preview": { @@ -238,10 +238,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 25.00, - "search_context_size_medium": 27.50, - "search_context_size_high": 30.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.030 } }, "gpt-4o-mini-search-preview": { @@ -263,10 +263,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 25.00, - "search_context_size_medium": 27.50, - "search_context_size_high": 30.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.030 } }, "gpt-4o-mini-2024-07-18": { @@ -287,7 +287,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "search_context_cost_per_1k_calls": { + "search_context_cost_per_query": { "search_context_size_low": 30.00, "search_context_size_medium": 35.00, "search_context_size_high": 50.00 diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 33d01523e1..6992e1afcf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16,7 +16,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { + "search_context_cost_per_query": { "search_context_size_low": 0.0000, "search_context_size_medium": 0.0000, "search_context_size_high": 0.0000 @@ -82,10 +82,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 30.00, - "search_context_size_medium": 35.00, - "search_context_size_high": 50.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.030, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.050 } }, "gpt-4o-search-preview": { @@ -107,10 +107,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 30.00, - "search_context_size_medium": 35.00, - "search_context_size_high": 50.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.030, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.050 } }, "gpt-4.5-preview": { @@ -238,10 +238,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 25.00, - "search_context_size_medium": 27.50, - "search_context_size_high": 30.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.030 } }, "gpt-4o-mini-search-preview": { @@ -263,10 +263,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_web_search": true, - "search_context_cost_per_1k_calls": { - "search_context_size_low": 25.00, - "search_context_size_medium": 27.50, - "search_context_size_high": 30.00 + "search_context_cost_per_query": { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.030 } }, "gpt-4o-mini-2024-07-18": { @@ -287,7 +287,7 @@ "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true, - "search_context_cost_per_1k_calls": { + "search_context_cost_per_query": { "search_context_size_low": 30.00, "search_context_size_medium": 35.00, "search_context_size_high": 50.00 From 78c371d2e88afd7d1b0ef18d8d77e22469824931 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:08:57 -0700 Subject: [PATCH 14/16] search_context_cost_per_query test --- tests/local_testing/test_get_model_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index f885a61e78..4cca96481d 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -518,12 +518,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): ], }, }, - "search_context_cost_per_1k_calls": { + "search_context_cost_per_query": { "type": "object", "properties": { - "low_context": {"type": "number"}, - "medium_context": {"type": "number"}, - "high_context": {"type": "number"}, + "search_context_size_low": {"type": "number"}, + "search_context_size_medium": {"type": "number"}, + "search_context_size_high": {"type": "number"}, }, "additionalProperties": False, }, From 69c9a782b284045b74836aef744dd6c670e539c4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:32:22 -0700 Subject: [PATCH 15/16] add supports_web_search --- tests/local_testing/test_get_model_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 4cca96481d..1a0f6d7a8d 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -500,6 +500,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_tool_choice": {"type": "boolean"}, "supports_video_input": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, + "supports_web_search": {"type": "boolean"}, "tool_use_system_prompt_tokens": {"type": "number"}, "tpm": {"type": "number"}, "supported_endpoints": { From cae75fe46bdaff97fd243c632e594bb48b7845db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Mar 2025 13:44:44 -0700 Subject: [PATCH 16/16] fix supports_web_search --- .../model_prices_and_context_window_backup.json | 16 ++-------------- model_prices_and_context_window.json | 16 ++-------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6992e1afcf..c07607d2ba 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -105,13 +105,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.030, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.050 - } + "supports_tool_choice": true }, "gpt-4.5-preview": { "max_tokens": 16384, @@ -236,13 +230,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.030 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview": { "max_tokens": 16384, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6992e1afcf..c07607d2ba 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -105,13 +105,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.030, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.050 - } + "supports_tool_choice": true }, "gpt-4.5-preview": { "max_tokens": 16384, @@ -236,13 +230,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275, - "search_context_size_high": 0.030 - } + "supports_tool_choice": true }, "gpt-4o-mini-search-preview": { "max_tokens": 16384,