diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e4963a6f1a..09f1375ca8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -4,6 +4,8 @@ import time import traceback from typing import List, Literal, Optional, Tuple, Union +from pydantic import BaseModel + import litellm import litellm._logging from litellm import verbose_logger @@ -14,6 +16,8 @@ from litellm.litellm_core_utils.llm_cost_calc.google import ( cost_per_token as google_cost_per_token, ) from litellm.litellm_core_utils.llm_cost_calc.utils import _generic_cost_per_character +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS from litellm.utils import ( CallTypes, CostPerToken, @@ -469,7 +473,9 @@ def completion_cost( prompt_characters = 0 completion_tokens = 0 completion_characters = 0 - if completion_response is not None: + if completion_response is not None and isinstance( + completion_response, BaseModel + ): # get input/output tokens from completion_response prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0) completion_tokens = completion_response.get("usage", {}).get( @@ -654,6 +660,7 @@ def response_cost_calculator( ImageResponse, TranscriptionResponse, TextCompletionResponse, + HttpxBinaryResponseContent, ], model: str, custom_llm_provider: Optional[str], @@ -687,7 +694,8 @@ def response_cost_calculator( if cache_hit is not None and cache_hit is True: response_cost = 0.0 else: - response_object._hidden_params["optional_params"] = optional_params + if isinstance(response_object, BaseModel): + response_object._hidden_params["optional_params"] = optional_params if isinstance(response_object, ImageResponse): response_cost = completion_cost( completion_response=response_object, @@ -697,12 +705,11 @@ def response_cost_calculator( ) else: if ( - model in litellm.model_cost - and custom_pricing is not None - and custom_llm_provider is True + model in litellm.model_cost or custom_pricing is True ): # override defaults if custom pricing is set base_model = model # base_model defaults to None if not set on model_info + response_cost = completion_cost( completion_response=response_object, call_type=call_type, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4edbce5e15..4382a1fcb5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -24,6 +24,8 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, ) +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS from litellm.types.utils import ( CallTypes, EmbeddingResponse, @@ -521,33 +523,36 @@ class Logging: self.model_call_details["cache_hit"] = cache_hit ## if model in model cost map - log the response cost ## else set cost to None - verbose_logger.debug(f"Model={self.model};") if ( - result is not None - and ( + result is not None and self.stream is not True + ): # handle streaming separately + if ( isinstance(result, ModelResponse) or isinstance(result, EmbeddingResponse) or isinstance(result, ImageResponse) or isinstance(result, TranscriptionResponse) or isinstance(result, TextCompletionResponse) - ) - and self.stream != True - ): # handle streaming separately - self.model_call_details["response_cost"] = ( - litellm.response_cost_calculator( - response_object=result, - model=self.model, - cache_hit=self.model_call_details.get("cache_hit", False), - custom_llm_provider=self.model_call_details.get( - "custom_llm_provider", None - ), - base_model=_get_base_model_from_metadata( - model_call_details=self.model_call_details - ), - call_type=self.call_type, - optional_params=self.optional_params, + or isinstance(result, HttpxBinaryResponseContent) # tts + ): + custom_pricing = use_custom_pricing_for_model( + litellm_params=self.litellm_params + ) + self.model_call_details["response_cost"] = ( + litellm.response_cost_calculator( + response_object=result, + model=self.model, + cache_hit=self.model_call_details.get("cache_hit", False), + custom_llm_provider=self.model_call_details.get( + "custom_llm_provider", None + ), + base_model=_get_base_model_from_metadata( + model_call_details=self.model_call_details + ), + call_type=self.call_type, + optional_params=self.optional_params, + custom_pricing=custom_pricing, + ) ) - ) else: # streaming chunks + image gen. self.model_call_details["response_cost"] = None @@ -2003,3 +2008,14 @@ def get_custom_logger_compatible_class( if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore return None + + +def use_custom_pricing_for_model(litellm_params: dict) -> bool: + model_info: Optional[dict] = litellm_params.get("metadata", {}).get( + "model_info", {} + ) + if model_info is not None: + for k, v in model_info.items(): + if k in SPECIAL_MODEL_INFO_PARAMS: + return True + return False diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 7f4b86ec40..99f2cf16a6 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,12 +1,9 @@ model_list: - - model_name: "*" + - model_name: tts litellm_params: - model: "openai/*" - mock_response: "Hello world!" - -litellm_settings: - success_callback: ["langfuse"] - failure_callback: ["langfuse"] + model: openai/tts-1 + api_key: os.environ/OPENAI_API_KEY + input_cost_per_character: 0.000015, general_settings: alerting: ["slack"] diff --git a/litellm/types/router.py b/litellm/types/router.py index 78d516d6c7..fb2c82c978 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -324,7 +324,12 @@ class DeploymentTypedDict(TypedDict): litellm_params: LiteLLMParamsTypedDict -SPECIAL_MODEL_INFO_PARAMS = ["input_cost_per_token", "output_cost_per_token"] +SPECIAL_MODEL_INFO_PARAMS = [ + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_character", + "output_cost_per_character", +] class Deployment(BaseModel):