From aef3d89f0cc7b607d6f25c294b952cd9e3165328 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 8 May 2024 14:13:51 -0700 Subject: [PATCH 1/2] fix add cooldown_deployment alert_type --- litellm/integrations/slack_alerting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index c53daf36c3..07c3585f08 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -710,6 +710,7 @@ Model Info: "db_exceptions", "daily_reports", "new_model_added", + "cooldown_deployment", ], **kwargs, ): From 597b09598cee794dd8efe8f0b767efabd0784b83 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 8 May 2024 14:14:14 -0700 Subject: [PATCH 2/2] feat - send alert on cooling down a deploymeny --- litellm/router.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 610c94324d..f1ac0135af 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1862,6 +1862,10 @@ class Router: self.cache.set_cache( value=cached_value, key=cooldown_key, ttl=cooldown_time ) + + self.send_deployment_cooldown_alert( + deployment_id=deployment, exception_status=exception_status + ) else: self.failed_calls.set_cache( key=deployment, value=updated_fails, ttl=cooldown_time @@ -3384,6 +3388,39 @@ class Router: ) print("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") # noqa + def send_deployment_cooldown_alert( + self, deployment_id: str, exception_status: Union[str, int] + ): + try: + from litellm.proxy.proxy_server import proxy_logging_obj + + # trigger slack alert saying deployment is in cooldown + if ( + proxy_logging_obj is not None + and proxy_logging_obj.alerting is not None + and "slack" in proxy_logging_obj.alerting + ): + _deployment = self.get_deployment(model_id=deployment_id) + if _deployment is None: + return + + _litellm_params = _deployment["litellm_params"] + temp_litellm_params = copy.deepcopy(_litellm_params) + temp_litellm_params = dict(temp_litellm_params) + _model_name = _deployment.get("model_name", None) + _api_base = litellm.get_api_base( + model=_model_name, optional_params=temp_litellm_params + ) + asyncio.create_task( + proxy_logging_obj.slack_alerting_instance.send_alert( + message=f"Router: Cooling down deployment: {_api_base}, for {self.cooldown_time} seconds. Got exception: {str(exception_status)}", + alert_type="cooldown_deployment", + level="Low", + ) + ) + except Exception as e: + pass + def flush_cache(self): litellm.cache = None self.cache.flush_cache()