From ffb7f9f280b8bf6ddcbc801f2ffc8efa5146e58e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 15:26:32 -0700 Subject: [PATCH 1/6] add fallback_reports as slack alert --- docs/my-website/docs/proxy/alerting.md | 1 + litellm/proxy/_types.py | 1 + litellm/proxy/utils.py | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index b8e5ebe207..7841ace580 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -126,6 +126,7 @@ AlertType = Literal[ "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ec788471a4..5e0de6fb09 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -115,6 +115,7 @@ AlertType = Literal[ "new_model_added", "outage_alerts", "region_outage_alerts", + "fallback_reports", ] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 212f8a9c87..d1d17d0ef5 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -222,6 +222,7 @@ class ProxyLogging: "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts", From 4cdc5c2e253ab51e2a3905c1ceec3a1ec03343c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 16:07:08 -0700 Subject: [PATCH 2/6] feat add prometheus api to get data from endpoint --- .../prometheus_helpers/prometheus_api.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 litellm/integrations/prometheus_helpers/prometheus_api.py diff --git a/litellm/integrations/prometheus_helpers/prometheus_api.py b/litellm/integrations/prometheus_helpers/prometheus_api.py new file mode 100644 index 0000000000..186514d354 --- /dev/null +++ b/litellm/integrations/prometheus_helpers/prometheus_api.py @@ -0,0 +1,53 @@ +""" +Helper functions to query prometheus API +""" + +import asyncio +import os +import time + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + +PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL") +PROMETHEUS_SELECTED_INSTANCE = litellm.get_secret("PROMETHEUS_SELECTED_INSTANCE") +async_http_handler = AsyncHTTPHandler() + + +async def get_metric_from_prometheus( + metric_name: str, +): + # Get the start of the current day in Unix timestamp + if PROMETHEUS_URL is None: + raise ValueError( + "PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env" + ) + + start_of_day = int(time.time()) - (int(time.time()) % 86400) + query = metric_name + response = await async_http_handler.get( + f"{PROMETHEUS_URL}/api/v1/query", params={"query": query} + ) # End of the day + _json_response = response.json() + results = response.json()["data"]["result"] + return results + + +async def get_fallback_metric_from_prometheus(): + response_json = await get_metric_from_prometheus( + metric_name="llm_deployment_successful_fallbacks_total" + ) + + if not response_json: + return "No fallback data available." + + response_message = "" + for result in response_json: + result = response_json + metric = result["metric"] + value = int(float(result["value"][1])) # Convert value to integer + + primary_model = metric.get("primary_model", "Unknown") + fallback_model = metric.get("fallback_model", "Unknown") + response_message += f"`{value} successful fallback requests` with `primary model={primary_model} -> fallback model={fallback_model}`" + return response_message From 35de40e6eb1d922a25abfc9ca3274166c71ad0ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 16:08:36 -0700 Subject: [PATCH 3/6] add fallback_reports in slack alert types --- litellm/integrations/slack_alerting.py | 37 ++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 44d0e48871..bb6698d730 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -166,6 +166,7 @@ class SlackAlerting(CustomLogger): "db_exceptions", "daily_reports", "spend_reports", + "fallback_reports", "cooldown_deployment", "new_model_added", "outage_alerts", @@ -1702,7 +1703,7 @@ Model Info: alerting_metadata={}, ) except Exception as e: - verbose_proxy_logger.error("Error sending weekly spend report", e) + verbose_proxy_logger.error("Error sending weekly spend report %s", e) async def send_monthly_spend_report(self): """ """ @@ -1754,4 +1755,36 @@ Model Info: alerting_metadata={}, ) except Exception as e: - verbose_proxy_logger.error("Error sending weekly spend report", e) + verbose_proxy_logger.error("Error sending weekly spend report %s", e) + + async def send_fallback_stats_from_prometheus(self): + """ + Helper to send fallback statistics from prometheus server -> to slack + + This runs once per day and sends an overview of all the fallback statistics + """ + try: + from litellm.integrations.prometheus_helpers.prometheus_api import ( + get_fallback_metric_from_prometheus, + ) + + # call prometheuslogger. + falllback_success_info_prometheus = ( + await get_fallback_metric_from_prometheus() + ) + + fallback_message = ( + f"*Fallback Statistics:*\n{falllback_success_info_prometheus}" + ) + + await self.send_alert( + message=fallback_message, + level="Low", + alert_type="fallback_reports", + alerting_metadata={}, + ) + + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report %s", e) + + pass From 28b930c0b78818e4df8061ebb0392b840af1953e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 16:36:06 -0700 Subject: [PATCH 4/6] feat - use api to get prometheus api metrics --- .../prometheus_helpers/prometheus_api.py | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/prometheus_helpers/prometheus_api.py b/litellm/integrations/prometheus_helpers/prometheus_api.py index 186514d354..247171e579 100644 --- a/litellm/integrations/prometheus_helpers/prometheus_api.py +++ b/litellm/integrations/prometheus_helpers/prometheus_api.py @@ -7,6 +7,7 @@ import os import time import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL") @@ -23,31 +24,43 @@ async def get_metric_from_prometheus( "PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env" ) - start_of_day = int(time.time()) - (int(time.time()) % 86400) - query = metric_name + query = f"{metric_name}[24h]" + now = int(time.time()) response = await async_http_handler.get( - f"{PROMETHEUS_URL}/api/v1/query", params={"query": query} + f"{PROMETHEUS_URL}/api/v1/query", params={"query": query, "time": now} ) # End of the day _json_response = response.json() + verbose_logger.debug("json response from prometheus /query api %s", _json_response) results = response.json()["data"]["result"] return results async def get_fallback_metric_from_prometheus(): - response_json = await get_metric_from_prometheus( - metric_name="llm_deployment_successful_fallbacks_total" - ) - - if not response_json: - return "No fallback data available." - + """ + Gets fallback metrics from prometheus for the last 24 hours + """ response_message = "" - for result in response_json: - result = response_json - metric = result["metric"] - value = int(float(result["value"][1])) # Convert value to integer + relevant_metrics = [ + "llm_deployment_successful_fallbacks_total", + "llm_deployment_failed_fallbacks_total", + ] + for metric in relevant_metrics: + response_json = await get_metric_from_prometheus( + metric_name=metric, + ) - primary_model = metric.get("primary_model", "Unknown") - fallback_model = metric.get("fallback_model", "Unknown") - response_message += f"`{value} successful fallback requests` with `primary model={primary_model} -> fallback model={fallback_model}`" + if response_json: + verbose_logger.debug("response json %s", response_json) + for result in response_json: + verbose_logger.debug("result= %s", result) + metric = result["metric"] + metric_values = result["values"] + most_recent_value = metric_values[0] + + value = int(float(most_recent_value[1])) # Convert value to integer + primary_model = metric.get("primary_model", "Unknown") + fallback_model = metric.get("fallback_model", "Unknown") + response_message += f"`{value} successful fallback requests` with primary model=`{primary_model}` -> fallback model=`{fallback_model}`" + response_message += "\n" + verbose_logger.debug("response message %s", response_message) return response_message From 017ba7a0fefaf09964ae8ad6192cf56b92388f02 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 16:37:59 -0700 Subject: [PATCH 5/6] add slack alerting on proxy_config.yaml --- litellm/proxy/proxy_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 8d499c1c0f..a6b23a389c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -35,6 +35,7 @@ default_vertex_config: general_settings: master_key: sk-1234 + alerting: ["slack"] litellm_settings: fallbacks: [{"gemini-1.5-pro-001": ["gpt-4o"]}] From aef13a37f498dd0b926e0f2f76709ad66647ed3e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 10 Aug 2024 16:56:03 -0700 Subject: [PATCH 6/6] feat add cron job for sending stats from prometheus --- litellm/proxy/proxy_server.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 419ac8f7d2..527fe03cf8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2795,6 +2795,19 @@ async def startup_event(): day=1, ) + # Beta Feature - only used when prometheus api is in .env + if os.getenv("PROMETHEUS_URL"): + from zoneinfo import ZoneInfo + + scheduler.add_job( + proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus, + "cron", + hour=9, + minute=0, + timezone=ZoneInfo("America/Los_Angeles"), # Pacific Time + ) + await proxy_logging_obj.slack_alerting_instance.send_fallback_stats_from_prometheus() + scheduler.start()