Merge pull request #5154 from BerriAI/litellm_send_prometheus_fallbacks_from_slack

[Feat-Proxy] send prometheus fallbacks stats to slack

docs/my-website/docs/proxy/alerting.md

@@ -126,6 +126,7 @@ AlertType = Literal[
     "db_exceptions",
     "daily_reports",
     "spend_reports",
+    "fallback_reports",
     "cooldown_deployment",
     "new_model_added",
     "outage_alerts",

litellm/integrations/prometheus_helpers/prometheus_api.py

@@ -0,0 +1,66 @@
+"""
+Helper functions to query prometheus API
+"""
+
+import asyncio
+import os
+import time
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL")
+PROMETHEUS_SELECTED_INSTANCE = litellm.get_secret("PROMETHEUS_SELECTED_INSTANCE")
+async_http_handler = AsyncHTTPHandler()
+
+
+async def get_metric_from_prometheus(
+    metric_name: str,
+):
+    # Get the start of the current day in Unix timestamp
+    if PROMETHEUS_URL is None:
+        raise ValueError(
+            "PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env"
+        )
+
+    query = f"{metric_name}[24h]"
+    now = int(time.time())
+    response = await async_http_handler.get(
+        f"{PROMETHEUS_URL}/api/v1/query", params={"query": query, "time": now}
+    )  # End of the day
+    _json_response = response.json()
+    verbose_logger.debug("json response from prometheus /query api %s", _json_response)
+    results = response.json()["data"]["result"]
+    return results
+
+
+async def get_fallback_metric_from_prometheus():
+    """
+    Gets fallback metrics from prometheus for the last 24 hours
+    """
+    response_message = ""
+    relevant_metrics = [
+        "llm_deployment_successful_fallbacks_total",
+        "llm_deployment_failed_fallbacks_total",
+    ]
+    for metric in relevant_metrics:
+        response_json = await get_metric_from_prometheus(
+            metric_name=metric,
+        )
+
+        if response_json:
+            verbose_logger.debug("response json %s", response_json)
+            for result in response_json:
+                verbose_logger.debug("result= %s", result)
+                metric = result["metric"]
+                metric_values = result["values"]
+                most_recent_value = metric_values[0]
+
+                value = int(float(most_recent_value[1]))  # Convert value to integer
+                primary_model = metric.get("primary_model", "Unknown")
+                fallback_model = metric.get("fallback_model", "Unknown")
+                response_message += f"`{value} successful fallback requests` with primary model=`{primary_model}` -> fallback model=`{fallback_model}`"
+                response_message += "\n"
+        verbose_logger.debug("response message %s", response_message)
+    return response_message

litellm/integrations/slack_alerting.py

@@ -166,6 +166,7 @@ class SlackAlerting(CustomLogger):
             "db_exceptions",
             "daily_reports",
             "spend_reports",
+            "fallback_reports",
             "cooldown_deployment",
             "new_model_added",
             "outage_alerts",
@@ -1702,7 +1703,7 @@ Model Info:
                 alerting_metadata={},
             )
         except Exception as e:
-            verbose_proxy_logger.error("Error sending weekly spend report", e)
+            verbose_proxy_logger.error("Error sending weekly spend report %s", e)
 
     async def send_monthly_spend_report(self):
         """ """
@@ -1754,4 +1755,36 @@ Model Info:
                 alerting_metadata={},
             )
         except Exception as e:
-            verbose_proxy_logger.error("Error sending weekly spend report", e)
+            verbose_proxy_logger.error("Error sending weekly spend report %s", e)
+
+    async def send_fallback_stats_from_prometheus(self):
+        """
+        Helper to send fallback statistics from prometheus server -> to slack
+
+        This runs once per day and sends an overview of all the fallback statistics
+        """
+        try:
+            from litellm.integrations.prometheus_helpers.prometheus_api import (
+                get_fallback_metric_from_prometheus,
+            )
+
+            # call prometheuslogger.
+            falllback_success_info_prometheus = (
+                await get_fallback_metric_from_prometheus()
+            )
+
+            fallback_message = (
+                f"*Fallback Statistics:*\n{falllback_success_info_prometheus}"
+            )
+
+            await self.send_alert(
+                message=fallback_message,
+                level="Low",
+                alert_type="fallback_reports",
+                alerting_metadata={},
+            )
+
+        except Exception as e:
+            verbose_proxy_logger.error("Error sending weekly spend report %s", e)
+
+        pass

litellm/proxy/_types.py

@@ -115,6 +115,7 @@ AlertType = Literal[
     "new_model_added",
     "outage_alerts",
     "region_outage_alerts",
+    "fallback_reports",
 ]
 
 

litellm/proxy/proxy_config.yaml

@@ -35,6 +35,7 @@ default_vertex_config:
 
 general_settings: 
   master_key: sk-1234
+  alerting: ["slack"]
 
 litellm_settings:
   fallbacks: [{"gemini-1.5-pro-001": ["gpt-4o"]}]

litellm/proxy/proxy_server.py

@@ -2795,6 +2795,19 @@ async def startup_event():
                 day=1,
             )
 
+            # Beta Feature - only used when prometheus api is in .env
+            if os.getenv("PROMETHEUS_URL"):
+                from zoneinfo import ZoneInfo
+
+                scheduler.add_job(
+                    proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus,
+                    "cron",
+                    hour=9,
+                    minute=0,
+                    timezone=ZoneInfo("America/Los_Angeles"),  # Pacific Time
+                )
+                await proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus()
+
         scheduler.start()
 
 

litellm/proxy/utils.py

@@ -222,6 +222,7 @@ class ProxyLogging:
             "db_exceptions",
             "daily_reports",
             "spend_reports",
+            "fallback_reports",
             "cooldown_deployment",
             "new_model_added",
             "outage_alerts",