Merge branch 'main' into litellm_ci_cd_linting_fixes_09_29_2025_p2

docs/my-website/release_notes/v1.77.3-stable/index.md

@@ -1,5 +1,5 @@
 ---
-title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
+title: "v1.77.3-stable - Priority Based Rate Limiting"
 slug: "v1-77-3"
 date: 2025-09-21T10:00:00
 authors:
@@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.77.3.rc.1
+ghcr.io/berriai/litellm:v1.77.3-stable
 ```
 
 </TabItem>
@@ -51,11 +51,27 @@ pip install litellm==1.77.3
 
 ## Priority Quota Reservation
 
+This release adds support for priority quota reservation. This allows Proxy Admins to reserve specific percentages of model capacity for different use cases. 
+ 
+This is great for use cases where you want to ensure your realtime use cases must always get priority responses and background development jobs can take longer. 
+
+<Image img={require('../../img/release_notes/quota.png')}  style={{ width: '800px', height: 'auto' }} />
+
+<br/>
+
 This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
 
 Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
 
-<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+## +550 RPS Performance Improvements
+
+<Image img={require('../../img/release_notes/perf_imp.png')}  style={{ width: '800px', height: 'auto' }} />
+
+<br/>
+
+This release delivers significant RPS improvements through targeted optimizations. 
+ 
+We've achieved a +500 RPS boost by fixing cache type inconsistencies that were causing frequent cache misses, plus an additional +50 RPS by removing unnecessary coroutine checks from the hot path. 
 
 
 ## New Models / Updated Models

litellm/proxy/proxy_server.py

@@ -637,11 +637,6 @@ async def proxy_startup_event(app: FastAPI):
         user_api_key_cache=user_api_key_cache,
     )
 
-    if use_background_health_checks:
-        asyncio.create_task(
-            _run_background_health_check()
-        )  # start the background health check coroutine.
-
     if prompt_injection_detection_obj is not None:  # [TODO] - REFACTOR THIS
         prompt_injection_detection_obj.update_environment(router=llm_router)
 
@@ -664,6 +659,12 @@ async def proxy_startup_event(app: FastAPI):
 
         await ProxyStartupEvent._update_default_team_member_budget()
 
+    # Start background health checks AFTER models are loaded and index is built
+    if use_background_health_checks:
+        asyncio.create_task(
+            _run_background_health_check()
+        )  # start the background health check coroutine.
+
     ## [Optional] Initialize dd tracer
     ProxyStartupEvent._init_dd_tracer()
 

litellm/router.py

@@ -409,7 +409,12 @@ class Router:
         )  # {"TEAM_ID": PatternMatchRouter}
         self.auto_routers: Dict[str, "AutoRouter"] = {}
 
+        # Initialize model ID to deployment index mapping for O(1) lookups
+        self.model_id_to_deployment_index_map: Dict[str, int] = {}
+        
         if model_list is not None:
+            # Build model index immediately to enable O(1) lookups from the start
+            self._build_model_id_to_deployment_index_map(model_list)
             model_list = copy.deepcopy(model_list)
             self.set_model_list(model_list)
             self.healthy_deployments: List = self.model_list  # type: ignore
@@ -4984,7 +4989,7 @@ class Router:
 
             model = deployment.to_json(exclude_none=True)
 
-            self.model_list.append(model)
+            self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id)
             return deployment
         except Exception as e:
             if self.ignore_invalid_deployments:
@@ -5095,6 +5100,7 @@ class Router:
     def set_model_list(self, model_list: list):
         original_model_list = copy.deepcopy(model_list)
         self.model_list = []
+        self.model_id_to_deployment_index_map = {}  # Reset the index
         # we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
 
         for model in original_model_list:
@@ -5334,10 +5340,42 @@ class Router:
         self._add_deployment(deployment=deployment)
 
         # add to model names
-        self.model_list.append(_deployment)
+        self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
         self.model_names.append(deployment.model_name)
         return deployment
 
+    def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: int) -> None:
+        """
+        Helper method to update deployment indices after a deployment has been removed from model_list.
+        
+        Parameters:
+        - model_id: str - the id of the deployment that was removed
+        - removal_idx: int - the index where the deployment was removed from model_list
+        """
+        # Update indices for all models after the removed one
+        for deployment_id, idx in self.model_id_to_deployment_index_map.items():
+            if idx > removal_idx:
+                self.model_id_to_deployment_index_map[deployment_id] = idx - 1
+        # Remove the deleted model from index
+        if model_id in self.model_id_to_deployment_index_map:
+            del self.model_id_to_deployment_index_map[model_id]
+
+
+    def _add_model_to_list_and_index_map(self, model: dict, model_id: Optional[str] = None) -> None:
+        """
+        Helper method to add a model to the model_list and update the model_id_to_deployment_index_map.
+        
+        Parameters:
+        - model: dict - the model to add to the list
+        - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
+        """
+        self.model_list.append(model)
+        # Update model index for O(1) lookup
+        if model_id is not None:
+            self.model_id_to_deployment_index_map[model_id] = len(self.model_list) - 1
+        elif model.get("model_info", {}).get("id") is not None:
+            self.model_id_to_deployment_index_map[model["model_info"]["id"]] = len(self.model_list) - 1
+
     def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]:
         """
         Add or update deployment
@@ -5363,12 +5401,15 @@ class Router:
                 # if there is a new litellm param -> then update the deployment
                 # remove the previous deployment
                 removal_idx: Optional[int] = None
-                for idx, model in enumerate(self.model_list):
-                    if model["model_info"]["id"] == deployment.model_info.id:
-                        removal_idx = idx
+                deployment_id = deployment.model_info.id
+                deployment_fast_mapping = self.model_id_to_deployment_index_map
+                
+                if deployment_id in deployment_fast_mapping:
+                    removal_idx = deployment_fast_mapping[deployment_id]
 
-                if removal_idx is not None:
-                    self.model_list.pop(removal_idx)
+                    if removal_idx is not None:
+                        self.model_list.pop(removal_idx)
+                        self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx)
 
             # if the model_id is not in router
             self.add_deployment(deployment=deployment)
@@ -5392,13 +5433,14 @@ class Router:
         - OR None (if deleted deployment not found)
         """
         deployment_idx = None
-        for idx, m in enumerate(self.model_list):
-            if m["model_info"]["id"] == id:
-                deployment_idx = idx
+        if id in self.model_id_to_deployment_index_map:
+            deployment_idx = self.model_id_to_deployment_index_map[id]
 
         try:
             if deployment_idx is not None:
+                # Pop the item from the list first
                 item = self.model_list.pop(deployment_idx)
+                self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx)
                 return item
             else:
                 return None
@@ -5411,15 +5453,17 @@ class Router:
 
         Raise Exception -> if model found in invalid format
         """
-        for model in self.model_list:
-            if "model_info" in model and "id" in model["model_info"]:
-                if model_id == model["model_info"]["id"]:
-                    if isinstance(model, dict):
-                        return Deployment(**model)
-                    elif isinstance(model, Deployment):
-                        return model
-                    else:
-                        raise Exception("Model invalid format - {}".format(type(model)))
+        # Use O(1) lookup via model_id_to_deployment_index_map only
+        if model_id in self.model_id_to_deployment_index_map:
+            idx = self.model_id_to_deployment_index_map[model_id]
+            model = self.model_list[idx]
+            if isinstance(model, dict):
+                return Deployment(**model)
+            elif isinstance(model, Deployment):
+                return model
+            else:
+                raise Exception("Model invalid format - {}".format(type(model)))
+        
         return None
 
     def get_deployment_credentials(self, model_id: str) -> Optional[dict]:
@@ -6037,6 +6081,31 @@ class Router:
                         additional_headers[header] = value
         return response
 
+    def _build_model_id_to_deployment_index_map(self, model_list: list):
+        """
+        Build model index from model list to enable O(1) lookups immediately.
+        This is called during initialization to avoid the race condition where
+        requests arrive before model_id_to_deployment_index_map is populated.
+        """
+        # First populate the model_list
+        self.model_list = []
+        for _, model in enumerate(model_list):
+            # Extract model_info from the model dict
+            model_info = model.get("model_info", {})
+            model_id = model_info.get("id")
+            
+            # If no ID exists, generate one using the same logic as set_model_list
+            if model_id is None:
+                model_name = model.get("model_name", "")
+                litellm_params = model.get("litellm_params", {})
+                model_id = self._generate_model_id(model_name, litellm_params)
+                # Update the model_info in the original list
+                if "model_info" not in model:
+                    model["model_info"] = {}
+                model["model_info"]["id"] = model_id
+            
+            self._add_model_to_list_and_index_map(model=model, model_id=model_id)
+
     def get_model_ids(
         self, model_name: Optional[str] = None, exclude_team_models: bool = False
     ) -> List[str]:

tests/local_testing/test_azure_openai.py

@@ -96,6 +96,6 @@ async def test_aaaaazure_tenant_id_auth(respx_mock: MockRouter):
 
         assert json_body == {
             "messages": [{"role": "user", "content": "Hello world!"}],
-            "model": "chatgpt-v-3",
+            "model": "gpt-4.1-nano",
             "stream": False,
         }

tests/local_testing/test_completion.py

@@ -712,7 +712,7 @@ def encode_image(image_path):
     "model",
     [
         "gpt-4o",
-        "azure/gpt-4o-new-test",
+        "azure/gpt-4.1-nano",
         "anthropic/claude-3-opus-20240229",
     ],
 )  #
@@ -3695,8 +3695,7 @@ def test_completion_volcengine():
     "model",
     [
         # "gemini-1.0-pro",
-        "gemini-1.5-pro",
-        # "gemini-2.5-flash-lite",
+        "gemini-2.5-flash-lite",
     ],
 )
 @pytest.mark.flaky(retries=3, delay=1)

tests/local_testing/test_custom_callback_input.py

@@ -774,7 +774,7 @@ async def test_async_embedding_openai():
         customHandler_failure = CompletionCustomHandler()
         litellm.callbacks = [customHandler_success]
         response = await litellm.aembedding(
-            model="azure/text-embedding-ada-002",
+            model="text-embedding-ada-002",
             input=["good morning from litellm"],
         )
         await asyncio.sleep(1)

tests/local_testing/test_embedding.py

@@ -380,8 +380,16 @@ def test_openai_azure_embedding_optional_arg():
             azure_ad_token="test",
         )
 
-        assert mock_client.called_once_with(model="test", input=["test"], timeout=600)
+        mock_client.assert_called_once_with(
+            model="test", 
+            input=["test"], 
+            extra_body={"azure_ad_token": "test"}, 
+            timeout=600, 
+            extra_headers={"X-Stainless-Raw-Response": "true"}
+        )
+        # Verify azure_ad_token is passed in extra_body, not as a direct parameter
         assert "azure_ad_token" not in mock_client.call_args.kwargs
+        assert mock_client.call_args.kwargs["extra_body"]["azure_ad_token"] == "test"
 
 
 # test_openai_azure_embedding()

tests/local_testing/test_exceptions.py

@@ -1349,7 +1349,7 @@ def test_context_window_exceeded_error_from_litellm_proxy():
 
 @pytest.mark.parametrize("sync_mode", [True, False])
 @pytest.mark.parametrize("stream_mode", [True, False])
-@pytest.mark.parametrize("model", ["azure/gpt-4o-new-test"])  # "gpt-4o-mini",
+@pytest.mark.parametrize("model", ["gpt-4.1-nano"])  # "gpt-4o-mini",
 @pytest.mark.asyncio
 async def test_exception_bubbling_up(sync_mode, stream_mode, model):
     """

tests/local_testing/test_function_calling.py

@@ -48,7 +48,7 @@ def get_current_weather(location, unit="fahrenheit"):
         "gpt-3.5-turbo-1106",
         "mistral/mistral-large-latest",
         "claude-3-haiku-20240307",
-        "gemini/gemini-1.5-pro",
+        "gemini/gemini-2.5-flash-lite",
         "anthropic.claude-3-sonnet-20240229-v1:0",
         "cohere_chat/command-r",
     ],

tests/local_testing/test_mock_request.py

@@ -155,15 +155,14 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
                 },
             },
             {
-                "model_name": "azure-gpt",
+                "model_name": "gpt-4.1-nano",
                 "litellm_params": {
-                    "model": "azure/gpt-4.1-nano",
-                    "api_key": os.getenv("AZURE_API_KEY"),
-                    "api_base": os.getenv("AZURE_API_BASE"),
+                    "model": "gpt-4.1-nano",
+                    "api_key": os.getenv("OPENAI_API_KEY"),
                 },
             },
         ],
-        fallbacks=[{"gpt-3.5-turbo": ["azure-gpt"]}],
+        fallbacks=[{"gpt-3.5-turbo": ["gpt-4.1-nano"]}],
     )
     response = router.completion(
         model="gpt-3.5-turbo",
@@ -176,5 +175,5 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
     end_time = time.time()
     assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}"
     assert (
-        "gpt-3.5-turbo-0125" in response.model
-    ), "Model should be azure gpt-3.5-turbo-0125"
+        "gpt-4.1-nano" in response.model
+    ), "Model should be gpt-4.1-nano"

tests/router_unit_tests/test_router_index_management.py

@@ -0,0 +1,105 @@
+import sys
+import os
+import pytest
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+from litellm import Router
+
+
+class TestRouterIndexManagement:
+    """Test cases for router index management functions"""
+
+    @pytest.fixture
+    def router(self):
+        """Create a router instance for testing"""
+        return Router(model_list=[])
+
+    def test_update_deployment_indices_after_removal(self, router):
+        """Test _update_deployment_indices_after_removal function"""
+        # Setup: Add models to router with proper structure
+        router.model_list = [
+            {"model": "test1", "model_info": {"id": "model-1"}}, 
+            {"model": "test2", "model_info": {"id": "model-2"}}, 
+            {"model": "test3", "model_info": {"id": "model-3"}}
+        ]
+        router.model_id_to_deployment_index_map = {"model-1": 0, "model-2": 1, "model-3": 2}
+
+        # Test: Remove model-2 (index 1)
+        router._update_deployment_indices_after_removal(model_id="model-2", removal_idx=1)
+
+        # Verify: model-2 is removed from index
+        assert "model-2" not in router.model_id_to_deployment_index_map
+        # Verify: model-3 index is updated (2 -> 1)
+        assert router.model_id_to_deployment_index_map["model-3"] == 1
+        # Verify: model-1 index remains unchanged
+        assert router.model_id_to_deployment_index_map["model-1"] == 0
+
+    def test_build_model_id_to_deployment_index_map(self, router):
+        """Test _build_model_id_to_deployment_index_map function"""
+        model_list = [
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo"},
+                "model_info": {"id": "model-1"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {"model": "gpt-4"},
+                "model_info": {"id": "model-2"},
+            },
+        ]
+
+        # Test: Build index from model list
+        router._build_model_id_to_deployment_index_map(model_list)
+
+        # Verify: model_list is populated
+        assert len(router.model_list) == 2
+        # Verify: model_id_to_deployment_index_map is correctly built
+        assert router.model_id_to_deployment_index_map["model-1"] == 0
+        assert router.model_id_to_deployment_index_map["model-2"] == 1
+
+    def test_add_model_to_list_and_index_map_from_model_info(self, router):
+        """Test _add_model_to_list_and_index_map extracting model_id from model_info"""
+        # Setup: Empty router
+        router.model_list = []
+        router.model_id_to_deployment_index_map = {}
+        
+        # Test: Add model without explicit model_id
+        model = {"model": "test-model", "model_info": {"id": "model-info-id"}}
+        router._add_model_to_list_and_index_map(model=model)
+        
+        # Verify: Model added to list
+        assert len(router.model_list) == 1
+        assert router.model_list[0] == model
+        
+        # Verify: Index map uses model_info.id
+        assert router.model_id_to_deployment_index_map["model-info-id"] == 0
+
+
+    def test_add_model_to_list_and_index_map_multiple_models(self, router):
+        """Test _add_model_to_list_and_index_map with multiple models to verify indexing"""
+        # Setup: Empty router
+        router.model_list = []
+        router.model_id_to_deployment_index_map = {}
+        
+        # Test: Add multiple models
+        model1 = {"model": "model1", "model_info": {"id": "id-1"}}
+        model2 = {"model": "model2", "model_info": {"id": "id-2"}}
+        model3 = {"model": "model3", "model_info": {"id": "id-3"}}
+        
+        router._add_model_to_list_and_index_map(model=model1, model_id="id-1")
+        router._add_model_to_list_and_index_map(model=model2, model_id="id-2")
+        router._add_model_to_list_and_index_map(model=model3, model_id="id-3")
+        
+        # Verify: All models added to list
+        assert len(router.model_list) == 3
+        assert router.model_list[0] == model1
+        assert router.model_list[1] == model2
+        assert router.model_list[2] == model3
+        
+        # Verify: Correct indices in map
+        assert router.model_id_to_deployment_index_map["id-1"] == 0
+        assert router.model_id_to_deployment_index_map["id-2"] == 1
+        assert router.model_id_to_deployment_index_map["id-3"] == 2