diff --git a/docs/my-website/img/release_notes/perf_imp.png b/docs/my-website/img/release_notes/perf_imp.png
new file mode 100644
index 0000000000..9fef6a6b2d
Binary files /dev/null and b/docs/my-website/img/release_notes/perf_imp.png differ
diff --git a/docs/my-website/img/release_notes/quota.png b/docs/my-website/img/release_notes/quota.png
new file mode 100644
index 0000000000..f8d15747f8
Binary files /dev/null and b/docs/my-website/img/release_notes/quota.png differ
diff --git a/docs/my-website/release_notes/v1.77.3-stable/index.md b/docs/my-website/release_notes/v1.77.3-stable/index.md
index adea34f978..c7c17e5bae 100644
--- a/docs/my-website/release_notes/v1.77.3-stable/index.md
+++ b/docs/my-website/release_notes/v1.77.3-stable/index.md
@@ -1,5 +1,5 @@
---
-title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
+title: "v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
@@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.77.3.rc.1
+ghcr.io/berriai/litellm:v1.77.3-stable
```
@@ -51,11 +51,27 @@ pip install litellm==1.77.3
## Priority Quota Reservation
+This release adds support for priority quota reservation. This allows Proxy Admins to reserve specific percentages of model capacity for different use cases.
+
+This is great for use cases where you want to ensure your realtime use cases must always get priority responses and background development jobs can take longer.
+
+
+
+
+
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
-
+## +550 RPS Performance Improvements
+
+
+
+
+
+This release delivers significant RPS improvements through targeted optimizations.
+
+We've achieved a +500 RPS boost by fixing cache type inconsistencies that were causing frequent cache misses, plus an additional +50 RPS by removing unnecessary coroutine checks from the hot path.
## New Models / Updated Models
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 6d2e56c444..f031b9ffbd 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -637,11 +637,6 @@ async def proxy_startup_event(app: FastAPI):
user_api_key_cache=user_api_key_cache,
)
- if use_background_health_checks:
- asyncio.create_task(
- _run_background_health_check()
- ) # start the background health check coroutine.
-
if prompt_injection_detection_obj is not None: # [TODO] - REFACTOR THIS
prompt_injection_detection_obj.update_environment(router=llm_router)
@@ -664,6 +659,12 @@ async def proxy_startup_event(app: FastAPI):
await ProxyStartupEvent._update_default_team_member_budget()
+ # Start background health checks AFTER models are loaded and index is built
+ if use_background_health_checks:
+ asyncio.create_task(
+ _run_background_health_check()
+ ) # start the background health check coroutine.
+
## [Optional] Initialize dd tracer
ProxyStartupEvent._init_dd_tracer()
diff --git a/litellm/router.py b/litellm/router.py
index 75b7835bb5..3cf99a4b21 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -409,7 +409,12 @@ class Router:
) # {"TEAM_ID": PatternMatchRouter}
self.auto_routers: Dict[str, "AutoRouter"] = {}
+ # Initialize model ID to deployment index mapping for O(1) lookups
+ self.model_id_to_deployment_index_map: Dict[str, int] = {}
+
if model_list is not None:
+ # Build model index immediately to enable O(1) lookups from the start
+ self._build_model_id_to_deployment_index_map(model_list)
model_list = copy.deepcopy(model_list)
self.set_model_list(model_list)
self.healthy_deployments: List = self.model_list # type: ignore
@@ -4984,7 +4989,7 @@ class Router:
model = deployment.to_json(exclude_none=True)
- self.model_list.append(model)
+ self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id)
return deployment
except Exception as e:
if self.ignore_invalid_deployments:
@@ -5095,6 +5100,7 @@ class Router:
def set_model_list(self, model_list: list):
original_model_list = copy.deepcopy(model_list)
self.model_list = []
+ self.model_id_to_deployment_index_map = {} # Reset the index
# we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
for model in original_model_list:
@@ -5334,10 +5340,42 @@ class Router:
self._add_deployment(deployment=deployment)
# add to model names
- self.model_list.append(_deployment)
+ self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
self.model_names.append(deployment.model_name)
return deployment
+ def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: int) -> None:
+ """
+ Helper method to update deployment indices after a deployment has been removed from model_list.
+
+ Parameters:
+ - model_id: str - the id of the deployment that was removed
+ - removal_idx: int - the index where the deployment was removed from model_list
+ """
+ # Update indices for all models after the removed one
+ for deployment_id, idx in self.model_id_to_deployment_index_map.items():
+ if idx > removal_idx:
+ self.model_id_to_deployment_index_map[deployment_id] = idx - 1
+ # Remove the deleted model from index
+ if model_id in self.model_id_to_deployment_index_map:
+ del self.model_id_to_deployment_index_map[model_id]
+
+
+ def _add_model_to_list_and_index_map(self, model: dict, model_id: Optional[str] = None) -> None:
+ """
+ Helper method to add a model to the model_list and update the model_id_to_deployment_index_map.
+
+ Parameters:
+ - model: dict - the model to add to the list
+ - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
+ """
+ self.model_list.append(model)
+ # Update model index for O(1) lookup
+ if model_id is not None:
+ self.model_id_to_deployment_index_map[model_id] = len(self.model_list) - 1
+ elif model.get("model_info", {}).get("id") is not None:
+ self.model_id_to_deployment_index_map[model["model_info"]["id"]] = len(self.model_list) - 1
+
def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]:
"""
Add or update deployment
@@ -5363,12 +5401,15 @@ class Router:
# if there is a new litellm param -> then update the deployment
# remove the previous deployment
removal_idx: Optional[int] = None
- for idx, model in enumerate(self.model_list):
- if model["model_info"]["id"] == deployment.model_info.id:
- removal_idx = idx
+ deployment_id = deployment.model_info.id
+ deployment_fast_mapping = self.model_id_to_deployment_index_map
+
+ if deployment_id in deployment_fast_mapping:
+ removal_idx = deployment_fast_mapping[deployment_id]
- if removal_idx is not None:
- self.model_list.pop(removal_idx)
+ if removal_idx is not None:
+ self.model_list.pop(removal_idx)
+ self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx)
# if the model_id is not in router
self.add_deployment(deployment=deployment)
@@ -5392,13 +5433,14 @@ class Router:
- OR None (if deleted deployment not found)
"""
deployment_idx = None
- for idx, m in enumerate(self.model_list):
- if m["model_info"]["id"] == id:
- deployment_idx = idx
+ if id in self.model_id_to_deployment_index_map:
+ deployment_idx = self.model_id_to_deployment_index_map[id]
try:
if deployment_idx is not None:
+ # Pop the item from the list first
item = self.model_list.pop(deployment_idx)
+ self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx)
return item
else:
return None
@@ -5411,15 +5453,17 @@ class Router:
Raise Exception -> if model found in invalid format
"""
- for model in self.model_list:
- if "model_info" in model and "id" in model["model_info"]:
- if model_id == model["model_info"]["id"]:
- if isinstance(model, dict):
- return Deployment(**model)
- elif isinstance(model, Deployment):
- return model
- else:
- raise Exception("Model invalid format - {}".format(type(model)))
+ # Use O(1) lookup via model_id_to_deployment_index_map only
+ if model_id in self.model_id_to_deployment_index_map:
+ idx = self.model_id_to_deployment_index_map[model_id]
+ model = self.model_list[idx]
+ if isinstance(model, dict):
+ return Deployment(**model)
+ elif isinstance(model, Deployment):
+ return model
+ else:
+ raise Exception("Model invalid format - {}".format(type(model)))
+
return None
def get_deployment_credentials(self, model_id: str) -> Optional[dict]:
@@ -6037,6 +6081,31 @@ class Router:
additional_headers[header] = value
return response
+ def _build_model_id_to_deployment_index_map(self, model_list: list):
+ """
+ Build model index from model list to enable O(1) lookups immediately.
+ This is called during initialization to avoid the race condition where
+ requests arrive before model_id_to_deployment_index_map is populated.
+ """
+ # First populate the model_list
+ self.model_list = []
+ for _, model in enumerate(model_list):
+ # Extract model_info from the model dict
+ model_info = model.get("model_info", {})
+ model_id = model_info.get("id")
+
+ # If no ID exists, generate one using the same logic as set_model_list
+ if model_id is None:
+ model_name = model.get("model_name", "")
+ litellm_params = model.get("litellm_params", {})
+ model_id = self._generate_model_id(model_name, litellm_params)
+ # Update the model_info in the original list
+ if "model_info" not in model:
+ model["model_info"] = {}
+ model["model_info"]["id"] = model_id
+
+ self._add_model_to_list_and_index_map(model=model, model_id=model_id)
+
def get_model_ids(
self, model_name: Optional[str] = None, exclude_team_models: bool = False
) -> List[str]:
diff --git a/tests/local_testing/test_azure_openai.py b/tests/local_testing/test_azure_openai.py
index 24f8500c44..0c6147b584 100644
--- a/tests/local_testing/test_azure_openai.py
+++ b/tests/local_testing/test_azure_openai.py
@@ -96,6 +96,6 @@ async def test_aaaaazure_tenant_id_auth(respx_mock: MockRouter):
assert json_body == {
"messages": [{"role": "user", "content": "Hello world!"}],
- "model": "chatgpt-v-3",
+ "model": "gpt-4.1-nano",
"stream": False,
}
diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py
index 17dc8929c6..b18a80da21 100644
--- a/tests/local_testing/test_completion.py
+++ b/tests/local_testing/test_completion.py
@@ -712,7 +712,7 @@ def encode_image(image_path):
"model",
[
"gpt-4o",
- "azure/gpt-4o-new-test",
+ "azure/gpt-4.1-nano",
"anthropic/claude-3-opus-20240229",
],
) #
@@ -3695,8 +3695,7 @@ def test_completion_volcengine():
"model",
[
# "gemini-1.0-pro",
- "gemini-1.5-pro",
- # "gemini-2.5-flash-lite",
+ "gemini-2.5-flash-lite",
],
)
@pytest.mark.flaky(retries=3, delay=1)
diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py
index 87e4d640f7..243883db9e 100644
--- a/tests/local_testing/test_custom_callback_input.py
+++ b/tests/local_testing/test_custom_callback_input.py
@@ -774,7 +774,7 @@ async def test_async_embedding_openai():
customHandler_failure = CompletionCustomHandler()
litellm.callbacks = [customHandler_success]
response = await litellm.aembedding(
- model="azure/text-embedding-ada-002",
+ model="text-embedding-ada-002",
input=["good morning from litellm"],
)
await asyncio.sleep(1)
diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py
index bf60fe23b8..60e2a0764b 100644
--- a/tests/local_testing/test_embedding.py
+++ b/tests/local_testing/test_embedding.py
@@ -380,8 +380,16 @@ def test_openai_azure_embedding_optional_arg():
azure_ad_token="test",
)
- assert mock_client.called_once_with(model="test", input=["test"], timeout=600)
+ mock_client.assert_called_once_with(
+ model="test",
+ input=["test"],
+ extra_body={"azure_ad_token": "test"},
+ timeout=600,
+ extra_headers={"X-Stainless-Raw-Response": "true"}
+ )
+ # Verify azure_ad_token is passed in extra_body, not as a direct parameter
assert "azure_ad_token" not in mock_client.call_args.kwargs
+ assert mock_client.call_args.kwargs["extra_body"]["azure_ad_token"] == "test"
# test_openai_azure_embedding()
diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py
index 3ac9ef68f0..bf21b59c45 100644
--- a/tests/local_testing/test_exceptions.py
+++ b/tests/local_testing/test_exceptions.py
@@ -1349,7 +1349,7 @@ def test_context_window_exceeded_error_from_litellm_proxy():
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize("stream_mode", [True, False])
-@pytest.mark.parametrize("model", ["azure/gpt-4o-new-test"]) # "gpt-4o-mini",
+@pytest.mark.parametrize("model", ["gpt-4.1-nano"]) # "gpt-4o-mini",
@pytest.mark.asyncio
async def test_exception_bubbling_up(sync_mode, stream_mode, model):
"""
diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py
index 71249898f7..1312c4dcc6 100644
--- a/tests/local_testing/test_function_calling.py
+++ b/tests/local_testing/test_function_calling.py
@@ -48,7 +48,7 @@ def get_current_weather(location, unit="fahrenheit"):
"gpt-3.5-turbo-1106",
"mistral/mistral-large-latest",
"claude-3-haiku-20240307",
- "gemini/gemini-1.5-pro",
+ "gemini/gemini-2.5-flash-lite",
"anthropic.claude-3-sonnet-20240229-v1:0",
"cohere_chat/command-r",
],
diff --git a/tests/local_testing/test_mock_request.py b/tests/local_testing/test_mock_request.py
index de8764edba..f959ed8038 100644
--- a/tests/local_testing/test_mock_request.py
+++ b/tests/local_testing/test_mock_request.py
@@ -155,15 +155,14 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
},
},
{
- "model_name": "azure-gpt",
+ "model_name": "gpt-4.1-nano",
"litellm_params": {
- "model": "azure/gpt-4.1-nano",
- "api_key": os.getenv("AZURE_API_KEY"),
- "api_base": os.getenv("AZURE_API_BASE"),
+ "model": "gpt-4.1-nano",
+ "api_key": os.getenv("OPENAI_API_KEY"),
},
},
],
- fallbacks=[{"gpt-3.5-turbo": ["azure-gpt"]}],
+ fallbacks=[{"gpt-3.5-turbo": ["gpt-4.1-nano"]}],
)
response = router.completion(
model="gpt-3.5-turbo",
@@ -176,5 +175,5 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
end_time = time.time()
assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}"
assert (
- "gpt-3.5-turbo-0125" in response.model
- ), "Model should be azure gpt-3.5-turbo-0125"
+ "gpt-4.1-nano" in response.model
+ ), "Model should be gpt-4.1-nano"
diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py
new file mode 100644
index 0000000000..bdd52f66ba
--- /dev/null
+++ b/tests/router_unit_tests/test_router_index_management.py
@@ -0,0 +1,105 @@
+import sys
+import os
+import pytest
+
+sys.path.insert(
+ 0, os.path.abspath("../..")
+) # Adds the parent directory to the system path
+from litellm import Router
+
+
+class TestRouterIndexManagement:
+ """Test cases for router index management functions"""
+
+ @pytest.fixture
+ def router(self):
+ """Create a router instance for testing"""
+ return Router(model_list=[])
+
+ def test_update_deployment_indices_after_removal(self, router):
+ """Test _update_deployment_indices_after_removal function"""
+ # Setup: Add models to router with proper structure
+ router.model_list = [
+ {"model": "test1", "model_info": {"id": "model-1"}},
+ {"model": "test2", "model_info": {"id": "model-2"}},
+ {"model": "test3", "model_info": {"id": "model-3"}}
+ ]
+ router.model_id_to_deployment_index_map = {"model-1": 0, "model-2": 1, "model-3": 2}
+
+ # Test: Remove model-2 (index 1)
+ router._update_deployment_indices_after_removal(model_id="model-2", removal_idx=1)
+
+ # Verify: model-2 is removed from index
+ assert "model-2" not in router.model_id_to_deployment_index_map
+ # Verify: model-3 index is updated (2 -> 1)
+ assert router.model_id_to_deployment_index_map["model-3"] == 1
+ # Verify: model-1 index remains unchanged
+ assert router.model_id_to_deployment_index_map["model-1"] == 0
+
+ def test_build_model_id_to_deployment_index_map(self, router):
+ """Test _build_model_id_to_deployment_index_map function"""
+ model_list = [
+ {
+ "model_name": "gpt-3.5-turbo",
+ "litellm_params": {"model": "gpt-3.5-turbo"},
+ "model_info": {"id": "model-1"},
+ },
+ {
+ "model_name": "gpt-4",
+ "litellm_params": {"model": "gpt-4"},
+ "model_info": {"id": "model-2"},
+ },
+ ]
+
+ # Test: Build index from model list
+ router._build_model_id_to_deployment_index_map(model_list)
+
+ # Verify: model_list is populated
+ assert len(router.model_list) == 2
+ # Verify: model_id_to_deployment_index_map is correctly built
+ assert router.model_id_to_deployment_index_map["model-1"] == 0
+ assert router.model_id_to_deployment_index_map["model-2"] == 1
+
+ def test_add_model_to_list_and_index_map_from_model_info(self, router):
+ """Test _add_model_to_list_and_index_map extracting model_id from model_info"""
+ # Setup: Empty router
+ router.model_list = []
+ router.model_id_to_deployment_index_map = {}
+
+ # Test: Add model without explicit model_id
+ model = {"model": "test-model", "model_info": {"id": "model-info-id"}}
+ router._add_model_to_list_and_index_map(model=model)
+
+ # Verify: Model added to list
+ assert len(router.model_list) == 1
+ assert router.model_list[0] == model
+
+ # Verify: Index map uses model_info.id
+ assert router.model_id_to_deployment_index_map["model-info-id"] == 0
+
+
+ def test_add_model_to_list_and_index_map_multiple_models(self, router):
+ """Test _add_model_to_list_and_index_map with multiple models to verify indexing"""
+ # Setup: Empty router
+ router.model_list = []
+ router.model_id_to_deployment_index_map = {}
+
+ # Test: Add multiple models
+ model1 = {"model": "model1", "model_info": {"id": "id-1"}}
+ model2 = {"model": "model2", "model_info": {"id": "id-2"}}
+ model3 = {"model": "model3", "model_info": {"id": "id-3"}}
+
+ router._add_model_to_list_and_index_map(model=model1, model_id="id-1")
+ router._add_model_to_list_and_index_map(model=model2, model_id="id-2")
+ router._add_model_to_list_and_index_map(model=model3, model_id="id-3")
+
+ # Verify: All models added to list
+ assert len(router.model_list) == 3
+ assert router.model_list[0] == model1
+ assert router.model_list[1] == model2
+ assert router.model_list[2] == model3
+
+ # Verify: Correct indices in map
+ assert router.model_id_to_deployment_index_map["id-1"] == 0
+ assert router.model_id_to_deployment_index_map["id-2"] == 1
+ assert router.model_id_to_deployment_index_map["id-3"] == 2