tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-05 02:23:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix litellm_param based costing

Browse Source
This commit is contained in:
Sameer Kankute
2025-10-08 21:14:23 +05:30
parent 13703f289b
commit 85d4142845
5 changed files with 238 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/litellm_core_utils/litellm_logging.py
+1 -1
View File
@@ -81,12 +81,12 @@ from litellm.types.llms.openai import (
)
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.rerank import RerankResponse
from litellm.types.router import CustomPricingLiteLLMParams
from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostResponseTypes,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
GuardrailStatus,
litellm/router.py
+1 -2
View File
@@ -125,7 +125,6 @@ from litellm.types.router import (
AllowedFailsPolicy,
AssistantsTypedDict,
CredentialLiteLLMParams,
CustomPricingLiteLLMParams,
CustomRoutingStrategyBase,
Deployment,
DeploymentTypedDict,
@@ -145,7 +144,7 @@ from litellm.types.services import ServiceTypes
from litellm.types.utils import GenericBudgetConfigType, LiteLLMBatch
from litellm.types.utils import ModelInfo
from litellm.types.utils import ModelInfo as ModelMapInfo
from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage
from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage, CustomPricingLiteLLMParams
from litellm.utils import (
CustomStreamWrapper,
EmbeddingResponse,
litellm/types/router.py
+5 -16
View File
@@ -4,24 +4,20 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
from litellm._uuid import uuid
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Tuple, Union, get_type_hints
import httpx
from httpx import AsyncClient, Client
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
from pydantic import BaseModel, ConfigDict, Field
from typing_extensions import Required, TypedDict
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm._uuid import uuid
from ..exceptions import RateLimitError
from .completion import CompletionRequest
from .embedding import EmbeddingRequest
from .llms.openai import OpenAIFileObject
from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from .utils import ModelResponse, ProviderSpecificModelInfo
from .utils import CustomPricingLiteLLMParams, ModelResponse
class ConfigurableClientsideParamsCustomAuth(TypedDict):
@@ -166,14 +162,7 @@ class CredentialLiteLLMParams(BaseModel):
watsonx_region_name: Optional[str] = None
class CustomPricingLiteLLMParams(BaseModel):
## CUSTOM PRICING ##
input_cost_per_token: Optional[float] = None
output_cost_per_token: Optional[float] = None
input_cost_per_second: Optional[float] = None
output_cost_per_second: Optional[float] = None
input_cost_per_pixel: Optional[float] = None
output_cost_per_pixel: Optional[float] = None
class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
@@ -482,7 +471,7 @@ class Deployment(BaseModel):
def to_json(self, **kwargs):
try:
return self.model_dump(**kwargs) # noqa
except Exception as e:
except Exception:
# if using pydantic v1
return self.dict(**kwargs)
@@ -602,7 +591,7 @@ class ModelGroupInfo(BaseModel):
def __init__(self, **data):
for field_name, field_type in get_type_hints(self.__class__).items():
if field_type == bool and data.get(field_name) is None:
if field_type is bool and data.get(field_name) is None:
data[field_name] = False
super().__init__(**data)
litellm/types/utils.py
+46 -1
View File
@@ -2226,6 +2226,51 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
turn_off_message_logging: Optional[bool] # when true will not log messages
litellm_disabled_callbacks: Optional[List[str]]
class CustomPricingLiteLLMParams(BaseModel):
## CUSTOM PRICING ##
input_cost_per_token: Optional[float] = None
output_cost_per_token: Optional[float] = None
input_cost_per_second: Optional[float] = None
output_cost_per_second: Optional[float] = None
input_cost_per_pixel: Optional[float] = None
output_cost_per_pixel: Optional[float] = None
# Include all ModelInfoBase fields as optional
# This allows any model_info parameter to be set in litellm_params
input_cost_per_token_flex: Optional[float] = None
input_cost_per_token_priority: Optional[float] = None
cache_creation_input_token_cost: Optional[float] = None
cache_creation_input_token_cost_above_1hr: Optional[float] = None
cache_read_input_token_cost: Optional[float] = None
cache_read_input_token_cost_flex: Optional[float] = None
cache_read_input_token_cost_priority: Optional[float] = None
input_cost_per_character: Optional[float] = None
input_cost_per_audio_token: Optional[float] = None
input_cost_per_token_above_128k_tokens: Optional[float] = None
input_cost_per_token_above_200k_tokens: Optional[float] = None
input_cost_per_character_above_128k_tokens: Optional[float] = None
input_cost_per_query: Optional[float] = None
input_cost_per_image: Optional[float] = None
input_cost_per_audio_per_second: Optional[float] = None
input_cost_per_video_per_second: Optional[float] = None
input_cost_per_second: Optional[float] = None
input_cost_per_token_batches: Optional[float] = None
output_cost_per_token_batches: Optional[float] = None
output_cost_per_token_flex: Optional[float] = None
output_cost_per_token_priority: Optional[float] = None
output_cost_per_character: Optional[float] = None
output_cost_per_audio_token: Optional[float] = None
output_cost_per_token_above_128k_tokens: Optional[float] = None
output_cost_per_token_above_200k_tokens: Optional[float] = None
output_cost_per_character_above_128k_tokens: Optional[float] = None
output_cost_per_image: Optional[float] = None
output_cost_per_reasoning_token: Optional[float] = None
output_cost_per_video_per_second: Optional[float] = None
output_cost_per_audio_per_second: Optional[float] = None
output_cost_per_second: Optional[float] = None
search_context_cost_per_query: Optional[Dict[str, Any]] = None
citation_cost_per_token: Optional[float] = None
tiered_pricing: Optional[List[Dict[str, Any]]] = None
all_litellm_params = [
"metadata",
@@ -2325,7 +2370,7 @@ all_litellm_params = [
"litellm_session_id",
"use_litellm_proxy",
"prompt_label",
] + list(StandardCallbackDynamicParams.__annotations__.keys())
] + list(StandardCallbackDynamicParams.__annotations__.keys()) + list(CustomPricingLiteLLMParams.model_fields.keys())
class KeyGenerationConfig(TypedDict, total=False):
tests/pass_through_unit_tests/test_pass_through_unit_tests.py
+185
View File
@@ -418,3 +418,188 @@ def test_is_bedrock_agent_runtime_route():
is False
)
assert _is_bedrock_agent_runtime_route("/some/random/endpoint") is False
def test_init_kwargs_filters_pricing_params(mock_request, mock_user_api_key_dict):
"""
Test that pricing parameters are properly filtered out from the request body
and don't get sent to the provider API.
This ensures that custom pricing parameters like:
- cache_read_input_token_cost
- input_cost_per_token_batches
- output_cost_per_token_batches
- cache_creation_input_token_cost
etc. are removed from the request body before sending to provider.
Regression test for: LIT-1221
"""
request = mock_request()
# Create a parsed body with pricing parameters that should be filtered out
parsed_body = {
"model": "gpt-4",
"messages": [{"role": "user", "content": "test"}],
# Standard pricing params (should be filtered)
"input_cost_per_token": 0.00002,
"output_cost_per_token": 0.00002,
"input_cost_per_second": 0.00001,
"output_cost_per_second": 0.00001,
# Cache-related pricing params (should be filtered)
"cache_read_input_token_cost": 0.00005,
"cache_creation_input_token_cost": 0.00003,
"cache_creation_input_token_cost_above_1hr": 0.00004,
# Batch pricing params (should be filtered)
"input_cost_per_token_batches": 0.00005,
"output_cost_per_token_batches": 0.00006,
# Other pricing params (should be filtered)
"input_cost_per_audio_token": 0.00001,
"output_cost_per_audio_token": 0.00001,
"input_cost_per_character": 0.000001,
"output_cost_per_character": 0.000001,
"input_cost_per_image": 0.001,
"output_cost_per_image": 0.001,
# Tiered pricing
"tiered_pricing": [{"input_cost_per_token": 0.00001}],
# This should NOT be filtered (it's a valid OpenAI parameter)
"temperature": 0.7,
"max_tokens": 100,
}
passthrough_payload = PassthroughStandardLoggingPayload(
url="https://api.openai.com/v1/chat/completions",
request_body=parsed_body.copy(),
)
result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
request=request,
user_api_key_dict=mock_user_api_key_dict,
passthrough_logging_payload=passthrough_payload,
_parsed_body=parsed_body,
litellm_call_id="test-call-id",
logging_obj=LiteLLMLoggingObj(
model="gpt-4",
messages=[{"role": "user", "content": "test"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="test-call-id",
function_id="test-function-id",
),
)
# Verify pricing parameters were filtered out from parsed_body
assert "input_cost_per_token" not in parsed_body
assert "output_cost_per_token" not in parsed_body
assert "input_cost_per_second" not in parsed_body
assert "output_cost_per_second" not in parsed_body
assert "cache_read_input_token_cost" not in parsed_body
assert "cache_creation_input_token_cost" not in parsed_body
assert "cache_creation_input_token_cost_above_1hr" not in parsed_body
assert "input_cost_per_token_batches" not in parsed_body
assert "output_cost_per_token_batches" not in parsed_body
assert "input_cost_per_audio_token" not in parsed_body
assert "output_cost_per_audio_token" not in parsed_body
assert "input_cost_per_character" not in parsed_body
assert "output_cost_per_character" not in parsed_body
assert "input_cost_per_image" not in parsed_body
assert "output_cost_per_image" not in parsed_body
assert "tiered_pricing" not in parsed_body
# Verify valid OpenAI parameters remain in parsed_body
assert parsed_body["model"] == "gpt-4"
assert parsed_body["messages"] == [{"role": "user", "content": "test"}]
assert parsed_body["temperature"] == 0.7
assert parsed_body["max_tokens"] == 100
# Verify pricing parameters are stored in litellm_params for internal use
litellm_params = result["litellm_params"]
assert litellm_params["input_cost_per_token"] == 0.00002
assert litellm_params["output_cost_per_token"] == 0.00002
# Note: Other pricing params are also stored but we test the key ones that caused the regression
def test_custom_pricing_used_in_cost_calculation():
"""
Test that when custom pricing parameters are provided in litellm_params,
they are actually used for cost calculation.
This ensures that the custom pricing functionality works end-to-end:
1. Pricing params are stored in litellm_params
2. These params are used by completion_cost() to calculate costs
Regression test for: LIT-1221
"""
from litellm import completion_cost, Choices, Message, ModelResponse
from litellm.utils import Usage
# Create a mock response with usage
resp = ModelResponse(
id="chatcmpl-test-123",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content="This is a test response",
role="assistant",
),
)
],
created=1234567890,
model="gpt-4",
object="chat.completion",
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
# Test 1: Standard pricing (should use default model pricing)
standard_cost = completion_cost(
completion_response=resp,
model="gpt-4",
)
print(f"Standard cost: {standard_cost}")
# Test 2: Custom pricing via custom_cost_per_token parameter
custom_input_price = 0.00010 # $0.0001 per token
custom_output_price = 0.00020 # $0.0002 per token
custom_cost = completion_cost(
completion_response=resp,
custom_cost_per_token={
"input_cost_per_token": custom_input_price,
"output_cost_per_token": custom_output_price,
},
)
# Calculate expected cost
expected_custom_cost = (100 * custom_input_price) + (50 * custom_output_price)
print(f"Custom cost: {custom_cost}")
print(f"Expected custom cost: {expected_custom_cost}")
# Verify custom pricing is used (should match our calculation)
assert round(custom_cost, 10) == round(expected_custom_cost, 10)
# Verify custom cost is different from standard cost (unless prices happen to match)
# This confirms custom pricing is actually being applied
assert custom_cost != standard_cost, "Custom pricing should produce different cost than standard pricing"
# Test 3: Custom pricing with cache_read_input_token_cost and input_cost_per_token_batches
# This specifically tests the parameters that were causing the original issue
cache_cost = completion_cost(
completion_response=resp,
custom_cost_per_token={
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00002,
"cache_read_input_token_cost": 0.000005, # Should be accepted
"input_cost_per_token_batches": 0.000003, # Should be accepted
"output_cost_per_token_batches": 0.000004, # Should be accepted
},
)
# Basic validation that it doesn't throw an error and returns a number
assert isinstance(cache_cost, (int, float))
assert cache_cost >= 0
print(f"Cache-aware cost: {cache_cost}")
print("✅ Custom pricing parameters are correctly used in cost calculation")