tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-09 16:24:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Bug Fix] Parallel Request Limiter with /messages (#17426)

Browse Source 
* fix: use standard_logging_object for parallel request limiter

* fix test parallel request limtier
This commit is contained in:
Ishaan Jaff
2025-12-03 14:13:28 -08:00
committed by GitHub
parent 9b3d8302cf
commit 100cfc11ac
2 changed files with 24 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/proxy/hooks/parallel_request_limiter_v3.py
+18 -16
View File
@@ -1343,19 +1343,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"INSIDE parallel request limiter ASYNC SUCCESS LOGGING"
)
# Get metadata from kwargs
litellm_metadata = kwargs["litellm_params"].get(
get_metadata_variable_name_from_kwargs(kwargs), {}
# Get metadata from standard_logging_object - this correctly handles both
# 'metadata' and 'litellm_metadata' fields from litellm_params
standard_logging_object = kwargs.get("standard_logging_object") or {}
standard_logging_metadata = standard_logging_object.get("metadata") or {}
# user_api_key_hash is the same as user_api_key (it's the hash)
user_api_key = standard_logging_metadata.get("user_api_key_hash")
user_api_key_user_id = standard_logging_metadata.get("user_api_key_user_id")
user_api_key_team_id = standard_logging_metadata.get("user_api_key_team_id")
user_api_key_organization_id = standard_logging_metadata.get(
"user_api_key_org_id"
)
if litellm_metadata is None:
return
user_api_key = litellm_metadata.get("user_api_key")
user_api_key_user_id = litellm_metadata.get("user_api_key_user_id")
user_api_key_team_id = litellm_metadata.get("user_api_key_team_id")
user_api_key_organization_id = litellm_metadata.get(
"user_api_key_organization_id"
)
user_api_key_end_user_id = kwargs.get("user") or litellm_metadata.get(
user_api_key_end_user_id = kwargs.get("user") or standard_logging_metadata.get(
"user_api_key_end_user_id"
)
model_group = get_model_group_from_litellm_kwargs(kwargs)
@@ -1501,10 +1501,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
litellm_parent_otel_span: Union[
Span, None
] = _get_parent_otel_span_from_kwargs(kwargs)
litellm_metadata = kwargs["litellm_params"]["metadata"]
user_api_key = (
litellm_metadata.get("user_api_key") if litellm_metadata else None
)
# Get metadata from standard_logging_object - this correctly handles both
# 'metadata' and 'litellm_metadata' fields from litellm_params
standard_logging_object = kwargs.get("standard_logging_object") or {}
standard_logging_metadata = standard_logging_object.get("metadata") or {}
user_api_key = standard_logging_metadata.get("user_api_key_hash")
pipeline_operations: List[RedisPipelineIncrementOperation] = []
if user_api_key:
tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
+6 -4
View File
@@ -22,6 +22,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token
from litellm.types.utils import ModelResponse, Usage
class TimeController:
def __init__(self):
self._current = datetime.utcnow()
@@ -461,10 +462,11 @@ async def test_token_rate_limit_type_respected_v3(monkeypatch, token_rate_limit_
)
# Create mock kwargs for the success event
# Use standard_logging_object which is the canonical source for metadata
mock_kwargs = {
"litellm_params": {
"standard_logging_object": {
"metadata": {
"user_api_key": _api_key,
"user_api_key_hash": _api_key,
"user_api_key_user_id": None,
"user_api_key_team_id": None,
"user_api_key_end_user_id": None,
@@ -532,8 +534,8 @@ async def test_async_log_failure_event_v3():
internal_usage_cache=InternalUsageCache(local_cache)
)
# Mock kwargs with user_api_key
mock_kwargs = {"litellm_params": {"metadata": {"user_api_key": _api_key}}}
# Mock kwargs with user_api_key via standard_logging_object
mock_kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}}
# Capture pipeline operations
captured_ops = []