From 477c54184bda814745bb06d4fbe4900cfc816bd2 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 02:07:50 -0400 Subject: [PATCH] perf: avoid unconditional router lookups in success handler Replace bare _get_deployment_default_tpm/rpm_limit calls in the async_log_success_event condition with get_key_model_tpm/rpm_limit (model_name=model_group). The higher-level getters short-circuit on key/team metadata hits before ever reaching the router, so requests that don't use deployment defaults incur no extra router lookup. Remove the now-unused bare helper imports. Also fix invalid `int = None` type hints in test helper signatures to `Optional[int] = None`. Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/hooks/parallel_request_limiter.py | 12 ++++++++---- tests/test_litellm/proxy/auth/test_auth_utils.py | 3 ++- .../proxy/test_model_info_default_limits.py | 5 +++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index b0046fd035..49c6436c22 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -14,8 +14,6 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( - _get_deployment_default_rpm_limit, - _get_deployment_default_tpm_limit, get_key_model_rpm_limit, get_key_model_tpm_limit, ) @@ -548,8 +546,14 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "model_rpm_limit" in user_api_key_metadata or "model_tpm_limit" in user_api_key_metadata or user_api_key_model_max_budget is not None - or _get_deployment_default_tpm_limit(model_group) is not None - or _get_deployment_default_rpm_limit(model_group) is not None + or get_key_model_tpm_limit( + user_api_key_dict, model_name=model_group + ) + is not None + or get_key_model_rpm_limit( + user_api_key_dict, model_name=model_group + ) + is not None ) ): request_count_api_key = ( diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index d64f17e70f..2058f61cb0 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2,6 +2,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID extraction. """ +from typing import Optional from unittest.mock import MagicMock, patch from litellm.proxy._types import UserAPIKeyAuth @@ -317,7 +318,7 @@ def test_get_end_user_id_falls_back_to_deprecated_user_header_name(): assert result == "user-legacy" -def _make_deployment_dict(model_name: str, tpm: int = None, rpm: int = None) -> dict: +def _make_deployment_dict(model_name: str, tpm: Optional[int] = None, rpm: Optional[int] = None) -> dict: """Helper to build a minimal deployment dict as returned by router.get_model_list.""" litellm_params: dict = {"model": model_name} if tpm is not None: diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index 907f2390a8..8b85531785 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -3,6 +3,7 @@ Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set litellm_params are returned by the /model/info endpoint. """ +from typing import Optional from unittest.mock import MagicMock, patch import pytest @@ -13,8 +14,8 @@ from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo def _make_deployment( model_name: str, - default_tpm: int = None, - default_rpm: int = None, + default_tpm: Optional[int] = None, + default_rpm: Optional[int] = None, ) -> Deployment: params: dict = {"model": f"openai/{model_name}"} if default_tpm is not None: