diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index ffbcfc0f44..2ee0588f19 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -17,7 +17,7 @@ Quick summary: - async_log_success_event() fires on GET /v1/batches/{id} (batch completion) """ -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from fastapi import HTTPException from pydantic import BaseModel @@ -179,16 +179,13 @@ class _PROXY_BatchRateLimiter(CustomLogger): model_has_failures=False, ) - increments = cast( - List[Dict[Literal["requests", "tokens"], int]], - [ - { - "requests": batch_usage.request_count, - "tokens": batch_usage.total_tokens, - } - for _ in descriptors - ], - ) + increment: Dict[Literal["requests", "tokens"], int] = { + "requests": batch_usage.request_count, + "tokens": batch_usage.total_tokens, + } + increments: List[Dict[Literal["requests", "tokens"], int]] = [ + increment for _ in descriptors + ] rate_limit_response = ( await self.parallel_request_limiter.atomic_check_and_increment_by_n( diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 2cbb527284..f7c0592992 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -562,6 +562,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # If priority is NOT enforced (saturation below threshold) but # priority_descriptors exist, increment them for tracking only — no # check, no rollback. This matches the prior tracking semantics. + # + # Using the non-atomic should_rate_limit (instead of + # atomic_check_and_increment_by_n) is intentional here: we don't want + # to enforce the limit, we only want to bump the counter so the + # priority allocation has accurate usage when it later becomes + # enforced. The increment-then-check semantics of should_rate_limit + # are fine because we ignore the OVER_LIMIT response. if priority_descriptors and not should_enforce_priority: priority_tracking_response = await self.v3_limiter.should_rate_limit( descriptors=priority_descriptors, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index cd85843ef1..4497e64c17 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -754,17 +754,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): counter_key = self.create_rate_limit_keys( descriptor_key, descriptor_value, rlt ) + # Counter-key TTL and window_size are conceptually distinct + # ("how long the counter Redis key lives" vs "how long the + # sliding window is"). They happen to be equal today because + # we have no descriptor type that needs them apart, but they + # are kept as separate variables here so a future custom-TTL + # descriptor doesn't reintroduce a silent expiry bug. Both + # the Lua script and the in-memory fallback read these from + # their respective ARGV / meta slots. + ttl_seconds = int(window_size) + window_size_seconds = int(window_size) keys.extend([window_key, counter_key]) - # Per-descriptor 4-tuple: limit, increment, ttl, window_size. - # window_size is per-descriptor — descriptors may carry custom - # windows distinct from self.window_size, and the Lua script - # uses this slot to evaluate window expiry. + # Per-counter 4-tuple matches the Lua ARGV layout exactly: + # [limit, increment, ttl_seconds, window_size_seconds]. script_args.extend( [ int(limit_value), inc_amount, - int(window_size), - int(window_size), + ttl_seconds, + window_size_seconds, ] ) per_counter_meta.append( @@ -775,8 +783,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): "window_key": window_key, "counter_key": counter_key, "increment": inc_amount, - "ttl": int(window_size), - "window_size": int(window_size), + "ttl": ttl_seconds, + "window_size": window_size_seconds, } ) @@ -822,13 +830,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): raw: List[Any], per_counter_meta: List[Dict[str, Any]], ) -> RateLimitResponse: - """Convert Lua script return value to RateLimitResponse.""" + """Convert Lua script return value to RateLimitResponse. + + Indexing invariant: `per_counter_meta` and `KEYS` are parallel-indexed + at the COUNTER level, not the descriptor level. A descriptor with both + RPM and TPM limits emits two `(window_key, counter_key)` pairs and + two meta entries — one per counter. The Lua script's loop variable + `i` therefore enumerates counters, and the over-limit return tuple + `{1, i, ...}` carries a counter index that maps directly to + `per_counter_meta[i - 1]`. Keep these arrays parallel at the counter + level when modifying this code. + """ if not raw: return RateLimitResponse(overall_code="OK", statuses=[]) status_code = int(raw[0]) if status_code == 1: - # Over limit: { 1, descriptor_index (1-based), current_counter, limit } + # Over limit: { 1, counter_index (1-based), current_counter, limit } descriptor_index = int(raw[1]) - 1 current_counter = int(raw[2]) limit = int(raw[3])