mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-09 01:08:05 +00:00
perf: avoid unconditional router lookups in success handler
Replace bare _get_deployment_default_tpm/rpm_limit calls in the async_log_success_event condition with get_key_model_tpm/rpm_limit (model_name=model_group). The higher-level getters short-circuit on key/team metadata hits before ever reaching the router, so requests that don't use deployment defaults incur no extra router lookup. Remove the now-unused bare helper imports. Also fix invalid `int = None` type hints in test helper signatures to `Optional[int] = None`. Co-Authored-By: Claude (claude-sonnet-4-6) <noreply@anthropic.com>
This commit is contained in:
@@ -14,8 +14,6 @@ from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
|
||||
from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
_get_deployment_default_rpm_limit,
|
||||
_get_deployment_default_tpm_limit,
|
||||
get_key_model_rpm_limit,
|
||||
get_key_model_tpm_limit,
|
||||
)
|
||||
@@ -548,8 +546,14 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
||||
"model_rpm_limit" in user_api_key_metadata
|
||||
or "model_tpm_limit" in user_api_key_metadata
|
||||
or user_api_key_model_max_budget is not None
|
||||
or _get_deployment_default_tpm_limit(model_group) is not None
|
||||
or _get_deployment_default_rpm_limit(model_group) is not None
|
||||
or get_key_model_tpm_limit(
|
||||
user_api_key_dict, model_name=model_group
|
||||
)
|
||||
is not None
|
||||
or get_key_model_rpm_limit(
|
||||
user_api_key_dict, model_name=model_group
|
||||
)
|
||||
is not None
|
||||
)
|
||||
):
|
||||
request_count_api_key = (
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
Unit tests for auth_utils functions related to rate limiting and customer ID extraction.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
@@ -317,7 +318,7 @@ def test_get_end_user_id_falls_back_to_deprecated_user_header_name():
|
||||
assert result == "user-legacy"
|
||||
|
||||
|
||||
def _make_deployment_dict(model_name: str, tpm: int = None, rpm: int = None) -> dict:
|
||||
def _make_deployment_dict(model_name: str, tpm: Optional[int] = None, rpm: Optional[int] = None) -> dict:
|
||||
"""Helper to build a minimal deployment dict as returned by router.get_model_list."""
|
||||
litellm_params: dict = {"model": model_name}
|
||||
if tpm is not None:
|
||||
|
||||
@@ -3,6 +3,7 @@ Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set
|
||||
litellm_params are returned by the /model/info endpoint.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -13,8 +14,8 @@ from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
def _make_deployment(
|
||||
model_name: str,
|
||||
default_tpm: int = None,
|
||||
default_rpm: int = None,
|
||||
default_tpm: Optional[int] = None,
|
||||
default_rpm: Optional[int] = None,
|
||||
) -> Deployment:
|
||||
params: dict = {"model": f"openai/{model_name}"}
|
||||
if default_tpm is not None:
|
||||
|
||||
Reference in New Issue
Block a user