tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-17 02:23:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #24611 from Sameerlite/Sameerlite/order-fallback2

Browse Source 
feat(router): order-based fallback across deployment priority levels
This commit is contained in:
yuneng-jiang
2026-03-27 20:15:30 +05:30
committed by Sameer Kankute
parent 15f5dc38c4
commit d3568efad0
5 changed files with 455 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/proxy/load_balancing.md
+34 -8
View File
@@ -324,17 +324,43 @@ model_list:
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
order: 2 # 👈 Used when order=1 fails
```
:::important
The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`.
:::
### How order-based fallback works
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next.
If all order levels are exhausted, the router falls through to any configured [model-level fallbacks](#fallbacks).
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-primary
api_key: os.environ/AZURE_API_KEY
order: 1
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-secondary
api_key: os.environ/AZURE_API_KEY_2
order: 2
- model_name: gpt-4-fallback
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
router_settings:
fallbacks:
- gpt-4:
- gpt-4-fallback # tried after all order levels fail
```
The fallback chain for the above config: `order=1` → `order=2` → `gpt-4-fallback`.
For 429 (rate limit) errors specifically, the failed deployment is immediately placed on cooldown. If all `order=1` deployments are on cooldown, the router picks `order=2` deployments directly during retries without waiting for the fallback path.
### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases}
docs/my-website/docs/routing.md
+5 -10
View File
@@ -842,6 +842,8 @@ Traffic mirroring allows you to "mimic" production traffic to a secondary (silen
Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them.
When a request to an `order=1` deployment fails (connection error, 404, 429, etc.), the router automatically tries `order=2` deployments, then `order=3`, and so on. Each order level gets its own set of retries before escalating to the next. If all order levels are exhausted, the router falls through to any configured [fallbacks](#fallbacks).
<Tabs>
<TabItem value="sdk" label="SDK">
@@ -862,18 +864,14 @@ model_list = [
"litellm_params": {
"model": "azure/gpt-4-fallback",
"api_key": os.getenv("AZURE_API_KEY_2"),
"order": 2, # 👈 Used when order=1 is unavailable
"order": 2, # 👈 Tried when order=1 fails
},
},
]
router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work
router = Router(model_list=model_list)
```
:::important
The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router.
:::
</TabItem>
<TabItem value="proxy" label="PROXY">
@@ -889,10 +887,7 @@ model_list:
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
order: 2 # 👈 Tried when order=1 fails
```
</TabItem>
litellm/router.py
+70 -6
View File
@@ -5290,6 +5290,64 @@ class Router:
if "fallback_depth" not in input_kwargs:
input_kwargs["fallback_depth"] = 0
# ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list
# Skip for error types that have their own dedicated fallback handlers
_skip_order_fallback = isinstance(
e,
(litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError),
)
all_deployments = self._get_all_deployments(model_name=original_model_group)
_order_set: set = {
d.get("litellm_params", {}).get("order")
for d in all_deployments
if d.get("litellm_params", {}).get("order") is not None
}
order_values: list = sorted(_order_set)
if len(order_values) > 1 and not _skip_order_fallback:
# Determine which order levels have already been tried
current_target = kwargs.get("_target_order")
skip_up_to = (
current_target if current_target is not None else order_values[0]
)
# Build order-based fallback entries (skip already-tried levels)
order_fallback_entries: List = [
{"model": original_model_group, "_target_order": o}
for o in order_values
if o > skip_up_to
]
# Get external fallbacks — handle both standard and non-standard formats
external_fallback_group: Optional[List] = None
if fallbacks is not None and model_group is not None:
if _check_non_standard_fallback_format(fallbacks=fallbacks):
# Non-standard formats (e.g. ["claude-3-haiku"] or
# [{"model": "...", "messages": [...]}]) are passed through directly
external_fallback_group = fallbacks
else:
external_fallback_group, generic_idx = get_fallback_model_group(
fallbacks=fallbacks,
model_group=cast(str, model_group),
)
if external_fallback_group is None and generic_idx is not None:
external_fallback_group = fallbacks[generic_idx]["*"]
# Combined list: order fallbacks first, then external
combined_fallbacks = order_fallback_entries + (
external_fallback_group or []
)
if combined_fallbacks:
input_kwargs.update(
{
"fallback_model_group": combined_fallbacks,
"original_model_group": original_model_group,
}
)
response = await run_async_fallback(
*args,
**input_kwargs,
)
return response
try:
verbose_router_logger.info("Trying to fallback b/w models")
@@ -8886,12 +8944,6 @@ class Router:
if i not in invalid_model_indices
]
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
if len(_returned_deployments) > 0:
_returned_deployments = litellm.utils._get_order_filtered_deployments(
_returned_deployments
)
return _returned_deployments
def _get_model_from_alias(self, model: str) -> Optional[str]:
@@ -9140,6 +9192,12 @@ class Router:
),
)
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order = (request_kwargs or {}).pop("_target_order", None)
healthy_deployments = litellm.utils._get_order_filtered_deployments(
cast(List[Dict], healthy_deployments), target_order=_target_order
)
if len(healthy_deployments) == 0:
exception = await async_raise_no_deployment_exception(
litellm_router_instance=self,
@@ -9544,6 +9602,12 @@ class Router:
request_kwargs=request_kwargs,
)
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order = (request_kwargs or {}).pop("_target_order", None)
healthy_deployments = litellm.utils._get_order_filtered_deployments(
healthy_deployments, target_order=_target_order
)
if len(healthy_deployments) == 0:
model_ids = self.get_model_ids(model_name=model)
_cooldown_time = self.cooldown_cache.get_min_cooldown(
litellm/utils.py
+15 -1
View File
@@ -4866,7 +4866,21 @@ def calculate_max_parallel_requests(
return None
def _get_order_filtered_deployments(healthy_deployments: List[Dict]) -> List:
def _get_order_filtered_deployments(
healthy_deployments: List[Dict], target_order: Optional[int] = None
) -> List:
if target_order is not None:
filtered = [
d
for d in healthy_deployments
if d["litellm_params"].get("order") == target_order
]
if filtered:
return filtered
# target_order doesn't match any deployment (e.g., external fallback model) — return all
return healthy_deployments
# Default: pick min order group
min_order = min(
(
deployment["litellm_params"]["order"]
tests/test_litellm/test_router_order_fallback.py
+331
View File
@@ -0,0 +1,331 @@
"""
Tests for order-based fallback routing.
When deployments have `order` set in litellm_params, lower order deployments
should be tried first, and higher order deployments should be used as fallbacks
when lower order deployments fail.
"""
from typing import Optional
import pytest
from litellm import Router
from litellm.utils import _get_order_filtered_deployments
# ---------------------------------------------------------------------------
# Unit tests for _get_order_filtered_deployments
# ---------------------------------------------------------------------------
class TestGetOrderFilteredDeployments:
def _make_deployment(self, order: Optional[int], dep_id: str) -> dict:
params: dict = {"model": "gpt-4o", "api_key": "key"}
if order is not None:
params["order"] = order
return {
"model_name": "test-model",
"litellm_params": params,
"model_info": {"id": dep_id},
}
def test_returns_min_order_group(self):
deps = [
self._make_deployment(1, "a"),
self._make_deployment(2, "b"),
self._make_deployment(1, "c"),
]
result = _get_order_filtered_deployments(deps)
assert len(result) == 2
assert all(d["model_info"]["id"] in ("a", "c") for d in result)
def test_target_order_filters_to_exact_level(self):
deps = [
self._make_deployment(1, "a"),
self._make_deployment(2, "b"),
self._make_deployment(3, "c"),
]
result = _get_order_filtered_deployments(deps, target_order=2)
assert len(result) == 1
assert result[0]["model_info"]["id"] == "b"
def test_target_order_no_match_returns_all(self):
deps = [
self._make_deployment(1, "a"),
self._make_deployment(2, "b"),
]
result = _get_order_filtered_deployments(deps, target_order=99)
assert len(result) == 2
def test_no_order_set_returns_all(self):
deps = [
self._make_deployment(None, "a"),
self._make_deployment(None, "b"),
]
result = _get_order_filtered_deployments(deps)
assert len(result) == 2
def test_empty_list(self):
result = _get_order_filtered_deployments([])
assert result == []
def test_single_order_returns_all_with_that_order(self):
deps = [
self._make_deployment(1, "a"),
self._make_deployment(1, "b"),
]
result = _get_order_filtered_deployments(deps)
assert len(result) == 2
# ---------------------------------------------------------------------------
# Integration tests for order-based fallback in Router
# ---------------------------------------------------------------------------
def test_router_order_without_pre_call_checks():
"""Order filtering should work even when enable_pre_call_checks=False (default)."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "key",
"mock_response": "from order 1",
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "key",
"mock_response": "from order 2",
"order": 2,
},
"model_info": {"id": "2"},
},
],
num_retries=0,
enable_pre_call_checks=False,
)
for _ in range(20):
response = router.completion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
assert response._hidden_params["model_id"] == "1"
def test_router_order_no_fallback_when_healthy():
"""When order=1 is healthy, order=2 should never be used."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "key",
"mock_response": "from order 1",
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "key",
"mock_response": "from order 2",
"order": 2,
},
"model_info": {"id": "2"},
},
],
num_retries=0,
)
for _ in range(50):
response = router.completion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
assert response._hidden_params["model_id"] == "1"
@pytest.mark.asyncio
async def test_router_order_fallback_on_failure():
"""When order=1 fails, order=2 should be tried as fallback."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad-key",
"mock_response": Exception("connection error"),
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good-key",
"mock_response": "success from order 2",
"order": 2,
},
"model_info": {"id": "2"},
},
],
num_retries=0,
)
response = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
assert response._hidden_params["model_id"] == "2"
@pytest.mark.asyncio
async def test_router_order_fallback_three_levels():
"""When order=1 and order=2 both fail, order=3 should be tried."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail 1"),
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail 2"),
"order": 2,
},
"model_info": {"id": "2"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "success from order 3",
"order": 3,
},
"model_info": {"id": "3"},
},
],
num_retries=0,
)
response = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
assert response._hidden_params["model_id"] == "3"
@pytest.mark.asyncio
async def test_router_order_fallback_then_external_fallback():
"""When all order levels fail, external fallbacks should be tried."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail order 1"),
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail order 2"),
"order": 2,
},
"model_info": {"id": "2"},
},
{
"model_name": "fallback-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "success from external fallback",
},
"model_info": {"id": "fallback"},
},
],
fallbacks=[{"test-model": ["fallback-model"]}],
num_retries=0,
)
response = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
assert response._hidden_params["model_id"] == "fallback"
@pytest.mark.asyncio
async def test_router_order_fallback_with_non_standard_fallbacks():
"""Non-standard fallback formats (e.g. fallbacks=["model-name"]) passed
per-request should still be tried after all order levels are exhausted."""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail order 1"),
"order": 1,
},
"model_info": {"id": "1"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("fail order 2"),
"order": 2,
},
"model_info": {"id": "2"},
},
{
"model_name": "fallback-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "success from non-standard fallback",
},
"model_info": {"id": "fallback"},
},
],
num_retries=0,
)
response = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
fallbacks=["fallback-model"], # non-standard format, passed per-request
)
assert response._hidden_params["model_id"] == "fallback"