Merge pull request #1877 from BerriAI/litellm_aert_when_budget_tracking_fails

[Feat] Slack Alert when budget tracking fails

litellm/proxy/proxy_server.py

@@ -809,16 +809,15 @@ async def _PROXY_track_cost_callback(
         litellm_params = kwargs.get("litellm_params", {}) or {}
         proxy_server_request = litellm_params.get("proxy_server_request") or {}
         user_id = proxy_server_request.get("body", {}).get("user", None)
+        user_id = user_id or kwargs["litellm_params"]["metadata"].get(
+            "user_api_key_user_id", None
+        )
         if kwargs.get("response_cost", None) is not None:
             response_cost = kwargs["response_cost"]
             user_api_key = kwargs["litellm_params"]["metadata"].get(
                 "user_api_key", None
             )
 
-            user_id = user_id or kwargs["litellm_params"]["metadata"].get(
-                "user_api_key_user_id", None
-            )
-
             if kwargs.get("cache_hit", False) == True:
                 response_cost = 0.0
                 verbose_proxy_logger.info(
@@ -852,9 +851,21 @@ async def _PROXY_track_cost_callback(
                     f"Model not in litellm model cost map. Add custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing"
                 )
     except Exception as e:
-        verbose_proxy_logger.debug(
-            f"error in tracking cost callback - {traceback.format_exc}"
+        error_msg = f"error in tracking cost callback - {traceback.format_exc()}"
+        model = kwargs.get("model", "")
+        metadata = kwargs.get("litellm_params", {}).get("metadata", {})
+        error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n metadata: {metadata}\n"
+        user_id = user_id or "not-found"
+        asyncio.create_task(
+            proxy_logging_obj.budget_alerts(
+                user_max_budget=0,
+                user_current_spend=0,
+                type="failed_tracking",
+                user_info=user_id,
+                error_message=error_msg,
+            )
         )
+        verbose_proxy_logger.debug(f"error in tracking cost callback - {error_msg}")
 
 
 async def update_database(

litellm/proxy/utils.py

@@ -198,10 +198,17 @@ class ProxyLogging:
 
     async def budget_alerts(
         self,
-        type: Literal["token_budget", "user_budget", "user_and_proxy_budget"],
+        type: Literal[
+            "token_budget",
+            "user_budget",
+            "user_and_proxy_budget",
+            "failed_budgets",
+            "failed_tracking",
+        ],
         user_max_budget: float,
         user_current_spend: float,
         user_info=None,
+        error_message="",
     ):
         if self.alerting is None:
             # do nothing if alerting is not switched on
@@ -221,6 +228,15 @@ class ProxyLogging:
             max_budget = token_info["max_budget"]
             user_id = token_info["user_id"]
             user_info = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}"""
+        elif type == "failed_tracking":
+            user_id = str(user_info)
+            user_info = f"\nUser ID: {user_id}\n Error {error_message}"
+            message = "Failed Tracking Cost for" + user_info
+            await self.alerting_handler(
+                message=message,
+                level="High",
+            )
+            return
         else:
             user_info = str(user_info)
         # percent of max_budget left to spend