diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index b8e5ebe207..7841ace580 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -126,6 +126,7 @@ AlertType = Literal[ "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts", diff --git a/litellm/integrations/prometheus_helpers/prometheus_api.py b/litellm/integrations/prometheus_helpers/prometheus_api.py new file mode 100644 index 0000000000..247171e579 --- /dev/null +++ b/litellm/integrations/prometheus_helpers/prometheus_api.py @@ -0,0 +1,66 @@ +""" +Helper functions to query prometheus API +""" + +import asyncio +import os +import time + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + +PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL") +PROMETHEUS_SELECTED_INSTANCE = litellm.get_secret("PROMETHEUS_SELECTED_INSTANCE") +async_http_handler = AsyncHTTPHandler() + + +async def get_metric_from_prometheus( + metric_name: str, +): + # Get the start of the current day in Unix timestamp + if PROMETHEUS_URL is None: + raise ValueError( + "PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env" + ) + + query = f"{metric_name}[24h]" + now = int(time.time()) + response = await async_http_handler.get( + f"{PROMETHEUS_URL}/api/v1/query", params={"query": query, "time": now} + ) # End of the day + _json_response = response.json() + verbose_logger.debug("json response from prometheus /query api %s", _json_response) + results = response.json()["data"]["result"] + return results + + +async def get_fallback_metric_from_prometheus(): + """ + Gets fallback metrics from prometheus for the last 24 hours + """ + response_message = "" + relevant_metrics = [ + "llm_deployment_successful_fallbacks_total", + "llm_deployment_failed_fallbacks_total", + ] + for metric in relevant_metrics: + response_json = await get_metric_from_prometheus( + metric_name=metric, + ) + + if response_json: + verbose_logger.debug("response json %s", response_json) + for result in response_json: + verbose_logger.debug("result= %s", result) + metric = result["metric"] + metric_values = result["values"] + most_recent_value = metric_values[0] + + value = int(float(most_recent_value[1])) # Convert value to integer + primary_model = metric.get("primary_model", "Unknown") + fallback_model = metric.get("fallback_model", "Unknown") + response_message += f"`{value} successful fallback requests` with primary model=`{primary_model}` -> fallback model=`{fallback_model}`" + response_message += "\n" + verbose_logger.debug("response message %s", response_message) + return response_message diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 44d0e48871..bb6698d730 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -166,6 +166,7 @@ class SlackAlerting(CustomLogger): "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts", @@ -1702,7 +1703,7 @@ Model Info: alerting_metadata={}, ) except Exception as e: - verbose_proxy_logger.error("Error sending weekly spend report", e) + verbose_proxy_logger.error("Error sending weekly spend report %s", e) async def send_monthly_spend_report(self): """ """ @@ -1754,4 +1755,36 @@ Model Info: alerting_metadata={}, ) except Exception as e: - verbose_proxy_logger.error("Error sending weekly spend report", e) + verbose_proxy_logger.error("Error sending weekly spend report %s", e) + + async def send_fallback_stats_from_prometheus(self): + """ + Helper to send fallback statistics from prometheus server -> to slack + + This runs once per day and sends an overview of all the fallback statistics + """ + try: + from litellm.integrations.prometheus_helpers.prometheus_api import ( + get_fallback_metric_from_prometheus, + ) + + # call prometheuslogger. + falllback_success_info_prometheus = ( + await get_fallback_metric_from_prometheus() + ) + + fallback_message = ( + f"*Fallback Statistics:*\n{falllback_success_info_prometheus}" + ) + + await self.send_alert( + message=fallback_message, + level="Low", + alert_type="fallback_reports", + alerting_metadata={}, + ) + + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report %s", e) + + pass diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ec788471a4..5e0de6fb09 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -115,6 +115,7 @@ AlertType = Literal[ "new_model_added", "outage_alerts", "region_outage_alerts", + "fallback_reports", ] diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 8d499c1c0f..a6b23a389c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -35,6 +35,7 @@ default_vertex_config: general_settings: master_key: sk-1234 + alerting: ["slack"] litellm_settings: fallbacks: [{"gemini-1.5-pro-001": ["gpt-4o"]}] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 419ac8f7d2..527fe03cf8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2795,6 +2795,19 @@ async def startup_event(): day=1, ) + # Beta Feature - only used when prometheus api is in .env + if os.getenv("PROMETHEUS_URL"): + from zoneinfo import ZoneInfo + + scheduler.add_job( + proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus, + "cron", + hour=9, + minute=0, + timezone=ZoneInfo("America/Los_Angeles"), # Pacific Time + ) + await proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus() + scheduler.start() diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 212f8a9c87..d1d17d0ef5 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -222,6 +222,7 @@ class ProxyLogging: "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts",