From 4eaeec2a2dfcd645182fde7b559e4adc6faf7fc2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 1 Jun 2024 17:39:44 -0700 Subject: [PATCH] feat - set allowed fails policy --- litellm/tests/test_router_retries.py | 8 +++++++- litellm/types/router.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_router_retries.py b/litellm/tests/test_router_retries.py index 1c0c3e49f8..879af261ab 100644 --- a/litellm/tests/test_router_retries.py +++ b/litellm/tests/test_router_retries.py @@ -128,12 +128,17 @@ async def test_router_retries_errors(sync_mode, error_type): ["AuthenticationErrorRetries", "ContentPolicyViolationErrorRetries"], # ) async def test_router_retry_policy(error_type): - from litellm.router import RetryPolicy + from litellm.router import RetryPolicy, AllowedFailsPolicy retry_policy = RetryPolicy( ContentPolicyViolationErrorRetries=3, AuthenticationErrorRetries=0 ) + allowed_fails_policy = AllowedFailsPolicy( + ContentPolicyViolationErrorAllowedFails=1000, + RateLimitErrorAllowedFails=100, + ) + router = Router( model_list=[ { @@ -156,6 +161,7 @@ async def test_router_retry_policy(error_type): }, ], retry_policy=retry_policy, + allowed_fails_policy=allowed_fails_policy, ) customHandler = MyCustomHandler() diff --git a/litellm/types/router.py b/litellm/types/router.py index 4a1f4498c6..38ddef361e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -385,8 +385,8 @@ class RouterErrors(enum.Enum): class AllowedFailsPolicy(BaseModel): """ - Use this to set a custom number of allowed_fails for each exception type before cooling down a deployment - If RateLimitErrorRetries = 3, then 3 retries will be made for RateLimitError + Use this to set a custom number of allowed fails/minute before cooling down a deployment + If `AuthenticationErrorAllowedFails = 1000`, then 1000 AuthenticationError will be allowed before cooling down a deployment Mapping of Exception type to allowed_fails for each exception https://docs.litellm.ai/docs/exception_mapping