From 85d4142845d0f5ff7602640a4e5063cacf2c69dd Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 8 Oct 2025 21:14:23 +0530 Subject: [PATCH] Fix litellm_param based costing --- litellm/litellm_core_utils/litellm_logging.py | 2 +- litellm/router.py | 3 +- litellm/types/router.py | 21 +- litellm/types/utils.py | 47 ++++- .../test_pass_through_unit_tests.py | 185 ++++++++++++++++++ 5 files changed, 238 insertions(+), 20 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 696c67c44d..eafcab8855 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -81,12 +81,12 @@ from litellm.types.llms.openai import ( ) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.rerank import RerankResponse -from litellm.types.router import CustomPricingLiteLLMParams from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, + CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, GuardrailStatus, diff --git a/litellm/router.py b/litellm/router.py index ef7bcba635..50f6d7f1b6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -125,7 +125,6 @@ from litellm.types.router import ( AllowedFailsPolicy, AssistantsTypedDict, CredentialLiteLLMParams, - CustomPricingLiteLLMParams, CustomRoutingStrategyBase, Deployment, DeploymentTypedDict, @@ -145,7 +144,7 @@ from litellm.types.services import ServiceTypes from litellm.types.utils import GenericBudgetConfigType, LiteLLMBatch from litellm.types.utils import ModelInfo from litellm.types.utils import ModelInfo as ModelMapInfo -from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage +from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage, CustomPricingLiteLLMParams from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, diff --git a/litellm/types/router.py b/litellm/types/router.py index 62f1c13eb4..3482df19c3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,24 +4,20 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum -from litellm._uuid import uuid from dataclasses import dataclass from typing import Any, Dict, List, Literal, Optional, Tuple, Union, get_type_hints import httpx -from httpx import AsyncClient, Client -from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel, ConfigDict, Field from typing_extensions import Required, TypedDict -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm._uuid import uuid -from ..exceptions import RateLimitError from .completion import CompletionRequest from .embedding import EmbeddingRequest from .llms.openai import OpenAIFileObject from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES -from .utils import ModelResponse, ProviderSpecificModelInfo +from .utils import CustomPricingLiteLLMParams, ModelResponse class ConfigurableClientsideParamsCustomAuth(TypedDict): @@ -166,14 +162,7 @@ class CredentialLiteLLMParams(BaseModel): watsonx_region_name: Optional[str] = None -class CustomPricingLiteLLMParams(BaseModel): - ## CUSTOM PRICING ## - input_cost_per_token: Optional[float] = None - output_cost_per_token: Optional[float] = None - input_cost_per_second: Optional[float] = None - output_cost_per_second: Optional[float] = None - input_cost_per_pixel: Optional[float] = None - output_cost_per_pixel: Optional[float] = None + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): @@ -482,7 +471,7 @@ class Deployment(BaseModel): def to_json(self, **kwargs): try: return self.model_dump(**kwargs) # noqa - except Exception as e: + except Exception: # if using pydantic v1 return self.dict(**kwargs) @@ -602,7 +591,7 @@ class ModelGroupInfo(BaseModel): def __init__(self, **data): for field_name, field_type in get_type_hints(self.__class__).items(): - if field_type == bool and data.get(field_name) is None: + if field_type is bool and data.get(field_name) is None: data[field_name] = False super().__init__(**data) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cac3bdfb01..d481422430 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2226,6 +2226,51 @@ class StandardCallbackDynamicParams(TypedDict, total=False): turn_off_message_logging: Optional[bool] # when true will not log messages litellm_disabled_callbacks: Optional[List[str]] +class CustomPricingLiteLLMParams(BaseModel): + ## CUSTOM PRICING ## + input_cost_per_token: Optional[float] = None + output_cost_per_token: Optional[float] = None + input_cost_per_second: Optional[float] = None + output_cost_per_second: Optional[float] = None + input_cost_per_pixel: Optional[float] = None + output_cost_per_pixel: Optional[float] = None + + # Include all ModelInfoBase fields as optional + # This allows any model_info parameter to be set in litellm_params + input_cost_per_token_flex: Optional[float] = None + input_cost_per_token_priority: Optional[float] = None + cache_creation_input_token_cost: Optional[float] = None + cache_creation_input_token_cost_above_1hr: Optional[float] = None + cache_read_input_token_cost: Optional[float] = None + cache_read_input_token_cost_flex: Optional[float] = None + cache_read_input_token_cost_priority: Optional[float] = None + input_cost_per_character: Optional[float] = None + input_cost_per_audio_token: Optional[float] = None + input_cost_per_token_above_128k_tokens: Optional[float] = None + input_cost_per_token_above_200k_tokens: Optional[float] = None + input_cost_per_character_above_128k_tokens: Optional[float] = None + input_cost_per_query: Optional[float] = None + input_cost_per_image: Optional[float] = None + input_cost_per_audio_per_second: Optional[float] = None + input_cost_per_video_per_second: Optional[float] = None + input_cost_per_second: Optional[float] = None + input_cost_per_token_batches: Optional[float] = None + output_cost_per_token_batches: Optional[float] = None + output_cost_per_token_flex: Optional[float] = None + output_cost_per_token_priority: Optional[float] = None + output_cost_per_character: Optional[float] = None + output_cost_per_audio_token: Optional[float] = None + output_cost_per_token_above_128k_tokens: Optional[float] = None + output_cost_per_token_above_200k_tokens: Optional[float] = None + output_cost_per_character_above_128k_tokens: Optional[float] = None + output_cost_per_image: Optional[float] = None + output_cost_per_reasoning_token: Optional[float] = None + output_cost_per_video_per_second: Optional[float] = None + output_cost_per_audio_per_second: Optional[float] = None + output_cost_per_second: Optional[float] = None + search_context_cost_per_query: Optional[Dict[str, Any]] = None + citation_cost_per_token: Optional[float] = None + tiered_pricing: Optional[List[Dict[str, Any]]] = None all_litellm_params = [ "metadata", @@ -2325,7 +2370,7 @@ all_litellm_params = [ "litellm_session_id", "use_litellm_proxy", "prompt_label", -] + list(StandardCallbackDynamicParams.__annotations__.keys()) +] + list(StandardCallbackDynamicParams.__annotations__.keys()) + list(CustomPricingLiteLLMParams.model_fields.keys()) class KeyGenerationConfig(TypedDict, total=False): diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 501fe24e65..445d0ca55e 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -418,3 +418,188 @@ def test_is_bedrock_agent_runtime_route(): is False ) assert _is_bedrock_agent_runtime_route("/some/random/endpoint") is False + + +def test_init_kwargs_filters_pricing_params(mock_request, mock_user_api_key_dict): + """ + Test that pricing parameters are properly filtered out from the request body + and don't get sent to the provider API. + + This ensures that custom pricing parameters like: + - cache_read_input_token_cost + - input_cost_per_token_batches + - output_cost_per_token_batches + - cache_creation_input_token_cost + etc. are removed from the request body before sending to provider. + + Regression test for: LIT-1221 + """ + request = mock_request() + + # Create a parsed body with pricing parameters that should be filtered out + parsed_body = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "test"}], + # Standard pricing params (should be filtered) + "input_cost_per_token": 0.00002, + "output_cost_per_token": 0.00002, + "input_cost_per_second": 0.00001, + "output_cost_per_second": 0.00001, + # Cache-related pricing params (should be filtered) + "cache_read_input_token_cost": 0.00005, + "cache_creation_input_token_cost": 0.00003, + "cache_creation_input_token_cost_above_1hr": 0.00004, + # Batch pricing params (should be filtered) + "input_cost_per_token_batches": 0.00005, + "output_cost_per_token_batches": 0.00006, + # Other pricing params (should be filtered) + "input_cost_per_audio_token": 0.00001, + "output_cost_per_audio_token": 0.00001, + "input_cost_per_character": 0.000001, + "output_cost_per_character": 0.000001, + "input_cost_per_image": 0.001, + "output_cost_per_image": 0.001, + # Tiered pricing + "tiered_pricing": [{"input_cost_per_token": 0.00001}], + # This should NOT be filtered (it's a valid OpenAI parameter) + "temperature": 0.7, + "max_tokens": 100, + } + + passthrough_payload = PassthroughStandardLoggingPayload( + url="https://api.openai.com/v1/chat/completions", + request_body=parsed_body.copy(), + ) + + result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=request, + user_api_key_dict=mock_user_api_key_dict, + passthrough_logging_payload=passthrough_payload, + _parsed_body=parsed_body, + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="gpt-4", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + ) + + # Verify pricing parameters were filtered out from parsed_body + assert "input_cost_per_token" not in parsed_body + assert "output_cost_per_token" not in parsed_body + assert "input_cost_per_second" not in parsed_body + assert "output_cost_per_second" not in parsed_body + assert "cache_read_input_token_cost" not in parsed_body + assert "cache_creation_input_token_cost" not in parsed_body + assert "cache_creation_input_token_cost_above_1hr" not in parsed_body + assert "input_cost_per_token_batches" not in parsed_body + assert "output_cost_per_token_batches" not in parsed_body + assert "input_cost_per_audio_token" not in parsed_body + assert "output_cost_per_audio_token" not in parsed_body + assert "input_cost_per_character" not in parsed_body + assert "output_cost_per_character" not in parsed_body + assert "input_cost_per_image" not in parsed_body + assert "output_cost_per_image" not in parsed_body + assert "tiered_pricing" not in parsed_body + + # Verify valid OpenAI parameters remain in parsed_body + assert parsed_body["model"] == "gpt-4" + assert parsed_body["messages"] == [{"role": "user", "content": "test"}] + assert parsed_body["temperature"] == 0.7 + assert parsed_body["max_tokens"] == 100 + + # Verify pricing parameters are stored in litellm_params for internal use + litellm_params = result["litellm_params"] + assert litellm_params["input_cost_per_token"] == 0.00002 + assert litellm_params["output_cost_per_token"] == 0.00002 + # Note: Other pricing params are also stored but we test the key ones that caused the regression + + +def test_custom_pricing_used_in_cost_calculation(): + """ + Test that when custom pricing parameters are provided in litellm_params, + they are actually used for cost calculation. + + This ensures that the custom pricing functionality works end-to-end: + 1. Pricing params are stored in litellm_params + 2. These params are used by completion_cost() to calculate costs + + Regression test for: LIT-1221 + """ + from litellm import completion_cost, Choices, Message, ModelResponse + from litellm.utils import Usage + + # Create a mock response with usage + resp = ModelResponse( + id="chatcmpl-test-123", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="This is a test response", + role="assistant", + ), + ) + ], + created=1234567890, + model="gpt-4", + object="chat.completion", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + ) + + # Test 1: Standard pricing (should use default model pricing) + standard_cost = completion_cost( + completion_response=resp, + model="gpt-4", + ) + print(f"Standard cost: {standard_cost}") + + # Test 2: Custom pricing via custom_cost_per_token parameter + custom_input_price = 0.00010 # $0.0001 per token + custom_output_price = 0.00020 # $0.0002 per token + + custom_cost = completion_cost( + completion_response=resp, + custom_cost_per_token={ + "input_cost_per_token": custom_input_price, + "output_cost_per_token": custom_output_price, + }, + ) + + # Calculate expected cost + expected_custom_cost = (100 * custom_input_price) + (50 * custom_output_price) + + print(f"Custom cost: {custom_cost}") + print(f"Expected custom cost: {expected_custom_cost}") + + # Verify custom pricing is used (should match our calculation) + assert round(custom_cost, 10) == round(expected_custom_cost, 10) + + # Verify custom cost is different from standard cost (unless prices happen to match) + # This confirms custom pricing is actually being applied + assert custom_cost != standard_cost, "Custom pricing should produce different cost than standard pricing" + + # Test 3: Custom pricing with cache_read_input_token_cost and input_cost_per_token_batches + # This specifically tests the parameters that were causing the original issue + cache_cost = completion_cost( + completion_response=resp, + custom_cost_per_token={ + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00002, + "cache_read_input_token_cost": 0.000005, # Should be accepted + "input_cost_per_token_batches": 0.000003, # Should be accepted + "output_cost_per_token_batches": 0.000004, # Should be accepted + }, + ) + + # Basic validation that it doesn't throw an error and returns a number + assert isinstance(cache_cost, (int, float)) + assert cache_cost >= 0 + + print(f"Cache-aware cost: {cache_cost}") + print("✅ Custom pricing parameters are correctly used in cost calculation")