From d2208023d20c447b8f1ee2daefabe3dcf7c5cff6 Mon Sep 17 00:00:00 2001 From: Sameerlite Date: Tue, 23 Sep 2025 10:48:27 +0530 Subject: [PATCH] Add service_tier based pricing support for openai --- litellm/cost_calculator.py | 15 +- litellm/litellm_core_utils/litellm_logging.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 68 +++++++- litellm/llms/openai/cost_calculation.py | 4 +- ...odel_prices_and_context_window_backup.json | 59 +++++++ litellm/types/utils.py | 12 ++ litellm/utils.py | 6 + model_prices_and_context_window.json | 61 +++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 163 ++++++++++++++++++ 9 files changed, 378 insertions(+), 11 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5d8f5faadf..36a562b357 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -148,6 +148,8 @@ def cost_per_token( # noqa: PLR0915 ### CALL TYPE ### call_type: CallTypesLiteral = "completion", audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds + ### SERVICE TIER ### + service_tier: Optional[str] = None, # for OpenAI service tier pricing ) -> Tuple[float, float]: # type: ignore """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -278,6 +280,7 @@ def cost_per_token( # noqa: PLR0915 model=model_without_prefix, usage=usage_block, custom_llm_provider=custom_llm_provider, + service_tier=service_tier, ) return prompt_cost, completion_cost @@ -327,7 +330,7 @@ def cost_per_token( # noqa: PLR0915 elif custom_llm_provider == "bedrock": return bedrock_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "openai": - return openai_cost_per_token(model=model, usage=usage_block) + return openai_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) elif custom_llm_provider == "databricks": return databricks_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "fireworks_ai": @@ -606,6 +609,8 @@ def completion_cost( # noqa: PLR0915 litellm_model_name: Optional[str] = None, router_model_id: Optional[str] = None, litellm_logging_obj: Optional[LitellmLoggingObject] = None, + ### SERVICE TIER ### + service_tier: Optional[str] = None, # for OpenAI service tier pricing ) -> float: """ Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. @@ -658,6 +663,10 @@ def completion_cost( # noqa: PLR0915 completion_response=completion_response ) rerank_billed_units: Optional[RerankBilledUnits] = None + + # Extract service_tier from optional_params if not provided directly + if service_tier is None and optional_params is not None: + service_tier = optional_params.get("service_tier") selected_model = _select_model_name_for_cost_calc( model=model, @@ -909,6 +918,7 @@ def completion_cost( # noqa: PLR0915 call_type=cast(CallTypesLiteral, call_type), audio_transcription_file_duration=audio_transcription_file_duration, rerank_billed_units=rerank_billed_units, + service_tier=service_tier, ) _final_cost = ( prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar @@ -1003,6 +1013,8 @@ def response_cost_calculator( litellm_model_name: Optional[str] = None, router_model_id: Optional[str] = None, litellm_logging_obj: Optional[LitellmLoggingObject] = None, + ### SERVICE TIER ### + service_tier: Optional[str] = None, # for OpenAI service tier pricing ) -> float: """ Returns @@ -1036,6 +1048,7 @@ def response_cost_calculator( litellm_model_name=litellm_model_name, router_model_id=router_model_id, litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, ) return response_cost except Exception as e: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 059fd9f1fc..65f665041f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1228,6 +1228,7 @@ class Logging(LiteLLMLoggingBaseClass): "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, "litellm_logging_obj": self, + "service_tier": self.optional_params.get("service_tier") if self.optional_params else None, } except Exception as e: # error creating kwargs for cost calculation debug_info = StandardLoggingModelCostFailureDebugInformation( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 60a3119841..1753ed2d1c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -4,7 +4,7 @@ from typing import Any, Literal, Optional, Tuple, TypedDict, cast import litellm -from litellm._logging import verbose_logger +from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, @@ -12,6 +12,7 @@ from litellm.types.utils import ( ModelInfo, PassthroughCallTypes, Usage, + ServiceTier, ) from litellm.utils import get_model_info @@ -114,8 +115,30 @@ def _generic_cost_per_character( return prompt_cost, completion_cost +def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str: + """ + Get the appropriate cost key based on service tier. + + Args: + base_key: The base cost key (e.g., "input_cost_per_token") + service_tier: The service tier ("flex", "priority", or None for standard) + + Returns: + str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token") + """ + if service_tier is None: + return base_key + + # Only use service tier specific keys for "flex" and "priority" + if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]: + return f"{base_key}_{service_tier.lower()}" + + # For any other service tier, use standard pricing + return base_key + + def _get_token_base_cost( - model_info: ModelInfo, usage: Usage + model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None ) -> Tuple[float, float, float, float, float]: """ Return prompt cost, completion cost, and cache costs for a given model and usage. @@ -126,21 +149,27 @@ def _get_token_base_cost( Returns: Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ + # Get service tier aware cost keys + input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier) + output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier) + cache_creation_cost_key = _get_service_tier_cost_key("cache_creation_input_token_cost", service_tier) + cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", service_tier) + prompt_base_cost = cast( - float, _get_cost_per_unit(model_info, "input_cost_per_token") + float, _get_cost_per_unit(model_info, input_cost_key) ) completion_base_cost = cast( - float, _get_cost_per_unit(model_info, "output_cost_per_token") + float, _get_cost_per_unit(model_info, output_cost_key) ) cache_creation_cost = cast( - float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost") + float, _get_cost_per_unit(model_info, cache_creation_cost_key) ) cache_creation_cost_above_1hr = cast( float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"), ) cache_read_cost = cast( - float, _get_cost_per_unit(model_info, "cache_read_input_token_cost") + float, _get_cost_per_unit(model_info, cache_read_cost_key) ) ## CHECK IF ABOVE THRESHOLD @@ -249,6 +278,29 @@ def _get_cost_per_unit( verbose_logger.exception( f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0" ) + + # If the service tier key doesn't exist or is None, try to fall back to the standard key + if cost_per_unit is None: + # Check if any service tier suffix exists in the cost key using ServiceTier enum + for service_tier in ServiceTier: + suffix = f"_{service_tier.value}" + if suffix in cost_key: + # Extract the base key by removing the matched suffix + base_key = cost_key.replace(suffix, '') + fallback_cost = model_info.get(base_key) + if isinstance(fallback_cost, float): + return fallback_cost + if isinstance(fallback_cost, int): + return float(fallback_cost) + if isinstance(fallback_cost, str): + try: + return float(fallback_cost) + except ValueError: + verbose_logger.exception( + f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0" + ) + break # Only try the first matching suffix + return default_value @@ -443,7 +495,7 @@ def _calculate_input_cost( def generic_cost_per_token( - model: str, usage: Usage, custom_llm_provider: str + model: str, usage: Usage, custom_llm_provider: str, service_tier: Optional[str] = None ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -495,7 +547,7 @@ def generic_cost_per_token( cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost, - ) = _get_token_base_cost(model_info=model_info, usage=usage) + ) = _get_token_base_cost(model_info=model_info, usage=usage, service_tier=service_tier) prompt_cost = _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 304c444e37..229f75f265 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -18,7 +18,7 @@ def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_sec return "cost_per_token" -def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: +def cost_per_token(model: str, usage: Usage, service_tier: Optional[str] = None) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -31,7 +31,7 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: """ ## CALCULATE INPUT COST return generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="openai" + model=model, usage=usage, custom_llm_provider="openai", service_tier=service_tier ) # ### Non-cached text tokens # non_cached_text_tokens = usage.prompt_tokens diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43ea3af320..f8f2bdf5ad 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11534,8 +11534,10 @@ }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11543,6 +11545,7 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11600,8 +11603,10 @@ }, "gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11609,6 +11614,7 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11666,8 +11672,10 @@ }, "gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_priority": 2e-07, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11675,6 +11683,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_priority": 8e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11773,8 +11782,10 @@ }, "gpt-4o": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_priority": 2.125e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_priority": 4.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -11782,6 +11793,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 1.7e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -11794,6 +11806,7 @@ "gpt-4o-2024-05-13": { "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_priority": 8.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, @@ -11801,6 +11814,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_priority": 2.625e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -11919,8 +11933,10 @@ }, "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_priority": 1.25e-07, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, + "input_cost_per_token_priority": 2.5e-07, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -11928,6 +11944,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, + "output_cost_per_token_priority": 1e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -12243,13 +12260,19 @@ }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12275,13 +12298,19 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12371,13 +12400,19 @@ }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, + "output_cost_per_token_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12403,13 +12438,19 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, + "output_cost_per_token_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12435,13 +12476,16 @@ }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12467,13 +12511,16 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -15177,13 +15224,19 @@ }, "o3": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_flex": 2.5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_flex": 4e-06, + "output_cost_per_token_priority": 1.4e-05, "supported_endpoints": [ "/v1/responses", "/v1/chat/completions", @@ -15399,13 +15452,19 @@ }, "o4-mini": { "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_flex": 1.38e-07, + "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_flex": 5.5e-07, + "input_cost_per_token_priority": 2e-06, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_flex": 2.2e-06, + "output_cost_per_token_priority": 8e-06, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_pdf_input": true, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 25f353b3bc..7b8ddf45c8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -122,9 +122,13 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): max_input_tokens: Required[Optional[int]] max_output_tokens: Required[Optional[int]] input_cost_per_token: Required[float] + input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing + input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost: Optional[float] cache_creation_input_token_cost_above_1hr: Optional[float] cache_read_input_token_cost: Optional[float] + cache_read_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing + cache_read_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing input_cost_per_character: Optional[float] # only for vertex ai models input_cost_per_audio_token: Optional[float] input_cost_per_token_above_128k_tokens: Optional[float] # only for vertex ai models @@ -142,6 +146,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_batches: Optional[float] output_cost_per_token_batches: Optional[float] output_cost_per_token: Required[float] + output_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing + output_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing output_cost_per_character: Optional[float] # only for vertex ai models output_cost_per_audio_token: Optional[float] output_cost_per_token_above_128k_tokens: Optional[ @@ -2583,6 +2589,12 @@ class SpecialEnums(Enum): LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR = "litellm_proxy;model_id:{};generic_response_id:{}" # generic implementation of 'managed batches' - used for finetuning and any future work. +class ServiceTier(Enum): + """Enum for service tier types used in cost calculations.""" + FLEX = "flex" + PRIORITY = "priority" + + LLMResponseTypes = Union[ ModelResponse, EmbeddingResponse, diff --git a/litellm/utils.py b/litellm/utils.py index e4cbaec706..549a982696 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4874,12 +4874,16 @@ def _get_model_info_helper( # noqa: PLR0915 max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), input_cost_per_token=_input_cost_per_token, + input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), + input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), cache_creation_input_token_cost=_model_info.get( "cache_creation_input_token_cost", None ), cache_read_input_token_cost=_model_info.get( "cache_read_input_token_cost", None ), + cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), + cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -4904,6 +4908,8 @@ def _get_model_info_helper( # noqa: PLR0915 "output_cost_per_token_batches" ), output_cost_per_token=_output_cost_per_token, + output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), + output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), output_cost_per_audio_token=_model_info.get( "output_cost_per_audio_token", None ), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43ea3af320..f33bcb2a3c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11534,8 +11534,10 @@ }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11543,6 +11545,7 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11600,8 +11603,10 @@ }, "gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11609,6 +11614,7 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11666,8 +11672,10 @@ }, "gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_priority": 2e-07, "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -11675,6 +11683,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_priority": 8e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11773,8 +11782,10 @@ }, "gpt-4o": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_priority": 2.125e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_priority": 4.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -11782,6 +11793,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 1.7e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -11794,6 +11806,7 @@ "gpt-4o-2024-05-13": { "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_priority": 8.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, @@ -11801,6 +11814,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_priority": 2.625e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -11919,8 +11933,10 @@ }, "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_priority": 1.25e-07, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, + "input_cost_per_token_priority": 2.5e-07, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -11928,6 +11944,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, + "output_cost_per_token_priority": 1e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -12243,13 +12260,19 @@ }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12275,13 +12298,19 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12303,6 +12332,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_service_tier": true, "supports_vision": true }, "gpt-5-chat": { @@ -12371,13 +12401,19 @@ }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, + "output_cost_per_token_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12403,13 +12439,19 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, + "output_cost_per_token_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12435,13 +12477,17 @@ }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_flex": 2.5e-08, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -12467,13 +12513,16 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -15177,13 +15226,19 @@ }, "o3": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_flex": 2.5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_flex": 4e-06, + "output_cost_per_token_priority": 1.4e-05, "supported_endpoints": [ "/v1/responses", "/v1/chat/completions", @@ -15399,13 +15454,19 @@ }, "o4-mini": { "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_flex": 1.375e-07, + "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_flex": 5.5e-07, + "input_cost_per_token_priority": 2e-06, "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_flex": 2.2e-06, + "output_cost_per_token_priority": 8e-06, "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_pdf_input": true, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index be3133ca37..0c7a6b2942 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -463,3 +463,166 @@ def test_calculate_cache_writing_cost(): ) assert result_zero == 0.0 + + +def test_service_tier_flex_pricing(): + """Test that flex service tier uses correct pricing (approximately 50% of standard).""" + # Set up environment for local model cost map + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Test with gpt-5-nano which has flex pricing + model = "gpt-5-nano" + custom_llm_provider = "openai" + + # Create usage object + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500 + ) + + # Test standard pricing + std_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier=None + ) + std_total = std_cost[0] + std_cost[1] + + # Test flex pricing + flex_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier="flex" + ) + flex_total = flex_cost[0] + flex_cost[1] + + # Verify flex is approximately 50% of standard + assert std_total > 0, "Standard cost should be greater than 0" + assert flex_total > 0, "Flex cost should be greater than 0" + + flex_ratio = flex_total / std_total + assert 0.45 <= flex_ratio <= 0.55, f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" + + # Verify specific costs match expected values + # gpt-5-nano flex: input=2.5e-08, output=2e-07 + expected_flex_prompt = 1000 * 2.5e-08 # 0.000025 + expected_flex_completion = 500 * 2e-07 # 0.0001 + expected_flex_total = expected_flex_prompt + expected_flex_completion + + assert abs(flex_cost[0] - expected_flex_prompt) < 1e-10, f"Flex prompt cost mismatch: {flex_cost[0]} vs {expected_flex_prompt}" + assert abs(flex_cost[1] - expected_flex_completion) < 1e-10, f"Flex completion cost mismatch: {flex_cost[1]} vs {expected_flex_completion}" + assert abs(flex_total - expected_flex_total) < 1e-10, f"Flex total cost mismatch: {flex_total} vs {expected_flex_total}" + + +def test_service_tier_default_pricing(): + """Test that when no service tier is provided, standard pricing is used.""" + # Set up environment for local model cost map + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Test with gpt-5-nano + model = "gpt-5-nano" + custom_llm_provider = "openai" + + # Create usage object + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500 + ) + + # Test with no service tier (should use standard) + default_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier=None + ) + + # Test with explicit standard service tier + standard_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier="standard" + ) + + # Both should be identical + assert abs(default_cost[0] - standard_cost[0]) < 1e-10, "Default and standard prompt costs should be identical" + assert abs(default_cost[1] - standard_cost[1]) < 1e-10, "Default and standard completion costs should be identical" + + # Verify specific costs match expected standard values + # gpt-5-nano standard: input=5e-08, output=4e-07 + expected_standard_prompt = 1000 * 5e-08 # 0.00005 + expected_standard_completion = 500 * 4e-07 # 0.0002 + expected_standard_total = expected_standard_prompt + expected_standard_completion + + assert abs(default_cost[0] - expected_standard_prompt) < 1e-10, f"Standard prompt cost mismatch: {default_cost[0]} vs {expected_standard_prompt}" + assert abs(default_cost[1] - expected_standard_completion) < 1e-10, f"Standard completion cost mismatch: {default_cost[1]} vs {expected_standard_completion}" + + +def test_service_tier_fallback_pricing(): + """Test that when service tier is provided but model doesn't have those keys, it falls back to standard pricing.""" + # Set up environment for local model cost map + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Test with gpt-4 which doesn't have flex pricing keys + model = "gpt-4" + custom_llm_provider = "openai" + + # Create usage object + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500 + ) + + # Test standard pricing + std_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier=None + ) + std_total = std_cost[0] + std_cost[1] + + # Test flex pricing (should fall back to standard since gpt-4 doesn't have flex keys) + flex_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier="flex" + ) + flex_total = flex_cost[0] + flex_cost[1] + + # Test priority pricing (should fall back to standard since gpt-4 doesn't have priority keys) + priority_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier="priority" + ) + priority_total = priority_cost[0] + priority_cost[1] + + # All should be identical (fallback to standard) + assert abs(std_total - flex_total) < 1e-10, f"Standard and flex costs should be identical (fallback): {std_total} vs {flex_total}" + assert abs(std_total - priority_total) < 1e-10, f"Standard and priority costs should be identical (fallback): {std_total} vs {priority_total}" + + # Verify costs are reasonable (not zero) + assert std_total > 0, "Standard cost should be greater than 0" + assert flex_total > 0, "Flex cost should be greater than 0 (fallback)" + assert priority_total > 0, "Priority cost should be greater than 0 (fallback)" + + # Verify specific costs match expected gpt-4 values + # gpt-4 standard: input=3e-05, output=6e-05 + expected_standard_prompt = 1000 * 3e-05 # 0.03 + expected_standard_completion = 500 * 6e-05 # 0.03 + expected_standard_total = expected_standard_prompt + expected_standard_completion + + assert abs(std_cost[0] - expected_standard_prompt) < 1e-10, f"Standard prompt cost mismatch: {std_cost[0]} vs {expected_standard_prompt}" + assert abs(std_cost[1] - expected_standard_completion) < 1e-10, f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}"