tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-16 14:23:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

(litellm SDK perf improvement) - use verbose_logger.debug and _cached_get_model_info_helper in _response_cost_calculator (#7720)

Browse Source 
* define _cached_get_model_info_helper

* use _cached_get_model_info_helper
This commit is contained in:
Ishaan Jaff
2025-01-12 15:27:54 -08:00
committed by GitHub
parent 15b52039d2
commit 6518bc70a0
2 changed files with 29 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/cost_calculator.py
+17 -9
View File
@@ -60,8 +60,7 @@ from litellm.utils import (
ModelResponse,
TextCompletionResponse,
TranscriptionResponse,
_get_model_info_helper,
print_verbose,
_cached_get_model_info_helper,
token_counter,
)
@@ -279,7 +278,7 @@ def cost_per_token( # noqa: PLR0915
elif custom_llm_provider == "deepseek":
return deepseek_cost_per_token(model=model, usage=usage_block)
else:
model_info = _get_model_info_helper(
model_info = _cached_get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider
)
@@ -292,8 +291,11 @@ def cost_per_token( # noqa: PLR0915
model_info.get("input_cost_per_second", None) is not None
and response_time_ms is not None
):
print_verbose(
f"For model={model} - input_cost_per_second: {model_info.get('input_cost_per_second')}; response time: {response_time_ms}"
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; response time: %s",
model,
model_info.get("input_cost_per_second", None),
response_time_ms,
)
## COST PER SECOND ##
prompt_tokens_cost_usd_dollar = (
@@ -308,16 +310,22 @@ def cost_per_token( # noqa: PLR0915
model_info.get("output_cost_per_second", None) is not None
and response_time_ms is not None
):
print_verbose(
f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; response time: {response_time_ms}"
verbose_logger.debug(
"For model=%s - output_cost_per_second: %s; response time: %s",
model,
model_info.get("output_cost_per_second", None),
response_time_ms,
)
## COST PER SECOND ##
completion_tokens_cost_usd_dollar = (
model_info["output_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
print_verbose(
f"Returned custom cost for model={model} - prompt_tokens_cost_usd_dollar: {prompt_tokens_cost_usd_dollar}, completion_tokens_cost_usd_dollar: {completion_tokens_cost_usd_dollar}"
verbose_logger.debug(
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
model,
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
)
return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
litellm/utils.py
+12
View File
@@ -4013,6 +4013,18 @@ def _get_max_position_embeddings(model_name: str) -> Optional[int]:
return None
@lru_cache(maxsize=16)
def _cached_get_model_info_helper(
model: str, custom_llm_provider: Optional[str]
) -> ModelInfoBase:
"""
_get_model_info_helper wrapped with lru_cache
Speed Optimization to hit high RPS
"""
return _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
def _get_model_info_helper( # noqa: PLR0915
model: str, custom_llm_provider: Optional[str] = None
) -> ModelInfoBase: