Include model name + configured TPM/RPM in priority rate-limit 429 er… (#27216)

* Include model name + configured TPM/RPM in priority rate-limit 429 errors (#27215) * Include model name + configured TPM/RPM in priority rate-limit 429 errors The current 429 message ('Priority-based rate limit exceeded. Priority: prod, Rate limit type: tokens, Remaining: -664145, Model saturation: 86.3%') doesn't tell the operator which model was hit or what the configured limit is, so they can't tell whether the priority allocation needs tuning or the model TPM is just too small. Add Model, Model TPM, and Model RPM to both the priority-based 429 and the sibling Model-capacity 429 in dynamic_rate_limiter_v3._check_rate_limits. Pure error-message change — no behavior or schema impact. * test: assert priority 429 includes model name + configured TPM/RPM Adds a regression test for the new fields in the priority-based 429 detail ('Model:', 'Model TPM:', 'Model RPM:'). Verified locally that the test fails against the unpatched dynamic_rate_limiter_v3.py and passes after the patch. --------- Co-authored-by: shin-watcher <ext-agent-shin@berri.ai> * Update litellm/proxy/hooks/dynamic_rate_limiter_v3.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update litellm/proxy/hooks/dynamic_rate_limiter_v3.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: shin-watcher <ext-agent-shin@berri.ai> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-08-03 08:23:16 +00:00 · 2026-05-05 19:05:22 -07:00
parent 73de892654
commit e9fb29061a
2 changed files with 100 additions and 0 deletions
@@ -498,6 +498,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
                            "error": f"Model capacity reached for {model}. "
                            f"Priority: {priority}, "
                            f"Rate limit type: {status['rate_limit_type']}, "
+                            f"Model TPM: {model_group_info.tpm if model_group_info.tpm is not None else 'not configured'}, "
+                            f"Model RPM: {model_group_info.rpm if model_group_info.rpm is not None else 'not configured'}, "
                            f"Remaining: {status['limit_remaining']}"
                        },
                        headers={
@@ -515,8 +517,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
                        status_code=429,
                        detail={
                            "error": f"Priority-based rate limit exceeded. "
+                            f"Model: {model}, "
                            f"Priority: {priority}, "
                            f"Rate limit type: {status['rate_limit_type']}, "
+                            f"Model TPM: {model_group_info.tpm if model_group_info.tpm is not None else 'not configured'}, "
+                            f"Model RPM: {model_group_info.rpm if model_group_info.rpm is not None else 'not configured'}, "
                            f"Remaining: {status['limit_remaining']}, "
                            f"Model saturation: {saturation:.1%}"
                        },
@@ -1677,3 +1677,98 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata():
    assert (
        "default_pool" not in priority_keys[0]
    ), f"Priority key should NOT use 'default_pool', should use team's priority. Got: {priority_keys[0]}"
+
+
+@pytest.mark.asyncio
+async def test_priority_429_includes_model_name_and_configured_limits():
+    """
+    The priority-based 429 should tell operators which model was hit and what
+    the model's configured TPM/RPM are, so they can decide whether to tune the
+    priority allocation or the model limits.
+
+    Regression test for the previous message that read:
+        "Priority-based rate limit exceeded. Priority: prod,
+         Rate limit type: tokens, Remaining: -664145,
+         Model saturation: 86.3%"
+    -- with no indication of which model was hit.
+    """
+    from fastapi import HTTPException
+
+    os.environ["LITELLM_LICENSE"] = "test-license-key"
+    litellm.priority_reservation = {"prod": 0.5}
+
+    dual_cache = DualCache()
+    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
+
+    model = "gpt-4o-test"
+    total_tpm = 1_000_000
+    total_rpm = 10_000
+
+    llm_router = Router(
+        model_list=[
+            {
+                "model_name": model,
+                "litellm_params": {
+                    "model": "gpt-3.5-turbo",
+                    "api_key": "test-key",
+                    "api_base": "test-base",
+                    "tpm": total_tpm,
+                    "rpm": total_rpm,
+                },
+            }
+        ]
+    )
+    handler.update_variables(llm_router=llm_router)
+
+    user = UserAPIKeyAuth()
+    user.metadata = {"priority": "prod"}
+    user.user_id = "prod_user"
+
+    model_group_info = handler.llm_router.get_model_group_info(model_group=model)
+
+    # Force the atomic check+increment to return OVER_LIMIT for the
+    # priority_model descriptor. saturation=0.95 keeps us above the
+    # default saturation threshold so priority limits are enforced.
+    over_limit_response = {
+        "overall_code": "OVER_LIMIT",
+        "statuses": [
+            {
+                "code": "OVER_LIMIT",
+                "descriptor_key": "priority_model",
+                "rate_limit_type": "tokens",
+                "limit_remaining": -664145,
+                "current_limit": int(total_tpm * 0.5),
+            }
+        ],
+    }
+
+    with patch.object(
+        handler.v3_limiter,
+        "atomic_check_and_increment_by_n",
+        new=AsyncMock(return_value=over_limit_response),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await handler._check_rate_limits(
+                model=model,
+                model_group_info=model_group_info,
+                user_api_key_dict=user,
+                priority="prod",
+                saturation=0.95,
+                data={"model": model},
+            )
+
+    assert exc_info.value.status_code == 429
+    detail = exc_info.value.detail
+    assert isinstance(detail, dict)
+    error_msg = detail["error"]
+
+    # New fields added by this change -- the whole point of the fix.
+    assert f"Model: {model}" in error_msg, error_msg
+    assert f"Model TPM: {total_tpm}" in error_msg, error_msg
+    assert f"Model RPM: {total_rpm}" in error_msg, error_msg
+
+    # Existing fields must still be present (no regression).
+    assert "Priority-based rate limit exceeded" in error_msg, error_msg
+    assert "Priority: prod" in error_msg, error_msg
+    assert "Rate limit type: tokens" in error_msg, error_msg
+    assert "Model saturation:" in error_msg, error_msg