From d3568efad07a65da97f319a572e576c4d9d4e58e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Mar 2026 22:06:20 -0700 Subject: [PATCH] Merge pull request #24611 from Sameerlite/Sameerlite/order-fallback2 feat(router): order-based fallback across deployment priority levels --- docs/my-website/docs/proxy/load_balancing.md | 42 ++- docs/my-website/docs/routing.md | 15 +- litellm/router.py | 76 +++- litellm/utils.py | 16 +- .../test_router_order_fallback.py | 331 ++++++++++++++++++ 5 files changed, 455 insertions(+), 25 deletions(-) create mode 100644 tests/test_litellm/test_router_order_fallback.py diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 897c04b2b0..93f3d94434 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -324,17 +324,43 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Used when order=1 fails ``` -:::important -The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`. -::: +### How order-based fallback works -If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. + +If all order levels are exhausted, the router falls through to any configured [model-level fallbacks](#fallbacks). + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-primary + api_key: os.environ/AZURE_API_KEY + order: 1 + + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-secondary + api_key: os.environ/AZURE_API_KEY_2 + order: 2 + + - model_name: gpt-4-fallback + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + +router_settings: + fallbacks: + - gpt-4: + - gpt-4-fallback # tried after all order levels fail +``` + +The fallback chain for the above config: `order=1` → `order=2` → `gpt-4-fallback`. + +For 429 (rate limit) errors specifically, the failed deployment is immediately placed on cooldown. If all `order=1` deployments are on cooldown, the router picks `order=2` deployments directly during retries without waiting for the fallback path. ### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 67e7f68114..5aa655ae21 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -842,6 +842,8 @@ Traffic mirroring allows you to "mimic" production traffic to a secondary (silen Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them. +When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. If all order levels are exhausted, the router falls through to any configured [fallbacks](#fallbacks). + @@ -862,18 +864,14 @@ model_list = [ "litellm_params": { "model": "azure/gpt-4-fallback", "api_key": os.getenv("AZURE_API_KEY_2"), - "order": 2, # 👈 Used when order=1 is unavailable + "order": 2, # 👈 Tried when order=1 fails }, }, ] -router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work +router = Router(model_list=model_list) ``` -:::important -The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router. -::: - @@ -889,10 +887,7 @@ model_list: litellm_params: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 - order: 2 # 👈 Used when order=1 is unavailable - -router_settings: - enable_pre_call_checks: true # 👈 Required for 'order' to work + order: 2 # 👈 Tried when order=1 fails ``` diff --git a/litellm/router.py b/litellm/router.py index 64ad6fc221..5cd4f83778 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5290,6 +5290,64 @@ class Router: if "fallback_depth" not in input_kwargs: input_kwargs["fallback_depth"] = 0 + # ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list + # Skip for error types that have their own dedicated fallback handlers + _skip_order_fallback = isinstance( + e, + (litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError), + ) + all_deployments = self._get_all_deployments(model_name=original_model_group) + _order_set: set = { + d.get("litellm_params", {}).get("order") + for d in all_deployments + if d.get("litellm_params", {}).get("order") is not None + } + order_values: list = sorted(_order_set) + if len(order_values) > 1 and not _skip_order_fallback: + # Determine which order levels have already been tried + current_target = kwargs.get("_target_order") + skip_up_to = ( + current_target if current_target is not None else order_values[0] + ) + # Build order-based fallback entries (skip already-tried levels) + order_fallback_entries: List = [ + {"model": original_model_group, "_target_order": o} + for o in order_values + if o > skip_up_to + ] + # Get external fallbacks — handle both standard and non-standard formats + external_fallback_group: Optional[List] = None + if fallbacks is not None and model_group is not None: + if _check_non_standard_fallback_format(fallbacks=fallbacks): + # Non-standard formats (e.g. ["claude-3-haiku"] or + # [{"model": "...", "messages": [...]}]) are passed through directly + external_fallback_group = fallbacks + else: + external_fallback_group, generic_idx = get_fallback_model_group( + fallbacks=fallbacks, + model_group=cast(str, model_group), + ) + if external_fallback_group is None and generic_idx is not None: + external_fallback_group = fallbacks[generic_idx]["*"] + + # Combined list: order fallbacks first, then external + combined_fallbacks = order_fallback_entries + ( + external_fallback_group or [] + ) + + if combined_fallbacks: + input_kwargs.update( + { + "fallback_model_group": combined_fallbacks, + "original_model_group": original_model_group, + } + ) + response = await run_async_fallback( + *args, + **input_kwargs, + ) + return response + try: verbose_router_logger.info("Trying to fallback b/w models") @@ -8886,12 +8944,6 @@ class Router: if i not in invalid_model_indices ] - ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - if len(_returned_deployments) > 0: - _returned_deployments = litellm.utils._get_order_filtered_deployments( - _returned_deployments - ) - return _returned_deployments def _get_model_from_alias(self, model: str) -> Optional[str]: @@ -9140,6 +9192,12 @@ class Router: ), ) + ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) + _target_order = (request_kwargs or {}).pop("_target_order", None) + healthy_deployments = litellm.utils._get_order_filtered_deployments( + cast(List[Dict], healthy_deployments), target_order=_target_order + ) + if len(healthy_deployments) == 0: exception = await async_raise_no_deployment_exception( litellm_router_instance=self, @@ -9544,6 +9602,12 @@ class Router: request_kwargs=request_kwargs, ) + ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) + _target_order = (request_kwargs or {}).pop("_target_order", None) + healthy_deployments = litellm.utils._get_order_filtered_deployments( + healthy_deployments, target_order=_target_order + ) + if len(healthy_deployments) == 0: model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( diff --git a/litellm/utils.py b/litellm/utils.py index 088ee07d63..0e3792773a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4866,7 +4866,21 @@ def calculate_max_parallel_requests( return None -def _get_order_filtered_deployments(healthy_deployments: List[Dict]) -> List: +def _get_order_filtered_deployments( + healthy_deployments: List[Dict], target_order: Optional[int] = None +) -> List: + if target_order is not None: + filtered = [ + d + for d in healthy_deployments + if d["litellm_params"].get("order") == target_order + ] + if filtered: + return filtered + # target_order doesn't match any deployment (e.g., external fallback model) — return all + return healthy_deployments + + # Default: pick min order group min_order = min( ( deployment["litellm_params"]["order"] diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py new file mode 100644 index 0000000000..760766a746 --- /dev/null +++ b/tests/test_litellm/test_router_order_fallback.py @@ -0,0 +1,331 @@ +""" +Tests for order-based fallback routing. + +When deployments have `order` set in litellm_params, lower order deployments +should be tried first, and higher order deployments should be used as fallbacks +when lower order deployments fail. +""" + +from typing import Optional + +import pytest + +from litellm import Router +from litellm.utils import _get_order_filtered_deployments + +# --------------------------------------------------------------------------- +# Unit tests for _get_order_filtered_deployments +# --------------------------------------------------------------------------- + + +class TestGetOrderFilteredDeployments: + def _make_deployment(self, order: Optional[int], dep_id: str) -> dict: + params: dict = {"model": "gpt-4o", "api_key": "key"} + if order is not None: + params["order"] = order + return { + "model_name": "test-model", + "litellm_params": params, + "model_info": {"id": dep_id}, + } + + def test_returns_min_order_group(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + self._make_deployment(1, "c"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + assert all(d["model_info"]["id"] in ("a", "c") for d in result) + + def test_target_order_filters_to_exact_level(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + self._make_deployment(3, "c"), + ] + result = _get_order_filtered_deployments(deps, target_order=2) + assert len(result) == 1 + assert result[0]["model_info"]["id"] == "b" + + def test_target_order_no_match_returns_all(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + ] + result = _get_order_filtered_deployments(deps, target_order=99) + assert len(result) == 2 + + def test_no_order_set_returns_all(self): + deps = [ + self._make_deployment(None, "a"), + self._make_deployment(None, "b"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + + def test_empty_list(self): + result = _get_order_filtered_deployments([]) + assert result == [] + + def test_single_order_returns_all_with_that_order(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(1, "b"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + + +# --------------------------------------------------------------------------- +# Integration tests for order-based fallback in Router +# --------------------------------------------------------------------------- + + +def test_router_order_without_pre_call_checks(): + """Order filtering should work even when enable_pre_call_checks=False (default).""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 1", + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + enable_pre_call_checks=False, + ) + + for _ in range(20): + response = router.completion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "1" + + +def test_router_order_no_fallback_when_healthy(): + """When order=1 is healthy, order=2 should never be used.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 1", + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + ) + + for _ in range(50): + response = router.completion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "1" + + +@pytest.mark.asyncio +async def test_router_order_fallback_on_failure(): + """When order=1 fails, order=2 should be tried as fallback.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad-key", + "mock_response": Exception("connection error"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good-key", + "mock_response": "success from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "2" + + +@pytest.mark.asyncio +async def test_router_order_fallback_three_levels(): + """When order=1 and order=2 both fail, order=3 should be tried.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from order 3", + "order": 3, + }, + "model_info": {"id": "3"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "3" + + +@pytest.mark.asyncio +async def test_router_order_fallback_then_external_fallback(): + """When all order levels fail, external fallbacks should be tried.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from external fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + fallbacks=[{"test-model": ["fallback-model"]}], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "fallback" + + +@pytest.mark.asyncio +async def test_router_order_fallback_with_non_standard_fallbacks(): + """Non-standard fallback formats (e.g. fallbacks=["model-name"]) passed + per-request should still be tried after all order levels are exhausted.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from non-standard fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + fallbacks=["fallback-model"], # non-standard format, passed per-request + ) + assert response._hidden_params["model_id"] == "fallback"