From 8462e8579258a93bcf6849f3e1f5e5e11f99d98c Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 7 Feb 2024 18:49:45 -0800 Subject: [PATCH 1/4] (feat) alert for failing cost tracking --- litellm/proxy/proxy_server.py | 14 +++++++++++++- litellm/proxy/utils.py | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f98ef9524..6c22efebc4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -852,8 +852,20 @@ async def _PROXY_track_cost_callback( f"Model not in litellm model cost map. Add custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing" ) except Exception as e: + error_msg = f"error in tracking cost callback - {traceback.format_exc()}" + error_msg += f"\n Args to _PROXY_track_cost_callback\n kwargs: {kwargs}\n completion_response: {completion_response}" + user_id = user_id or "not-found" + asyncio.create_task( + proxy_logging_obj.budget_alerts( + user_max_budget=0, + user_current_spend=0, + type="failed_tracking", + user_info=user_id, + error_message=error_msg, + ) + ) verbose_proxy_logger.debug( - f"error in tracking cost callback - {traceback.format_exc}" + f"error in tracking cost callback - {traceback.format_exc()}" ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8804028cc1..919799ca58 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -198,10 +198,17 @@ class ProxyLogging: async def budget_alerts( self, - type: Literal["token_budget", "user_budget", "user_and_proxy_budget"], + type: Literal[ + "token_budget", + "user_budget", + "user_and_proxy_budget", + "failed_budgets", + "failed_tracking", + ], user_max_budget: float, user_current_spend: float, user_info=None, + error_message="", ): if self.alerting is None: # do nothing if alerting is not switched on @@ -221,6 +228,15 @@ class ProxyLogging: max_budget = token_info["max_budget"] user_id = token_info["user_id"] user_info = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}""" + elif type == "failed_tracking": + user_id = str(user_info) + user_info = f"\nUser ID: {user_id}\n Error {error_message}" + message = "Failed Tracking Cost for" + user_info + await self.alerting_handler( + message=message, + level="High", + ) + return else: user_info = str(user_info) # percent of max_budget left to spend From 8a328b4c6d7f9a442d2a0ccccb8b52117636dbd2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 7 Feb 2024 19:20:15 -0800 Subject: [PATCH 2/4] (fix) track cost for semantic_caching, place on langfuse trace --- litellm/caching.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/litellm/caching.py b/litellm/caching.py index f996a58735..3522a9d436 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -427,10 +427,16 @@ class RedisSemanticCache(BaseCache): else [] ) if llm_router is not None and self.embedding_model in router_model_names: + user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") embedding_response = await llm_router.aembedding( model=self.embedding_model, input=prompt, cache={"no-store": True, "no-cache": True}, + metadata={ + "user_api_key": user_api_key, + "semantic-cache-embedding": True, + "trace_id": kwargs.get("metadata", {}).get("trace_id", None), + }, ) else: # convert to embedding @@ -476,13 +482,20 @@ class RedisSemanticCache(BaseCache): else [] ) if llm_router is not None and self.embedding_model in router_model_names: + user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") embedding_response = await llm_router.aembedding( model=self.embedding_model, input=prompt, cache={"no-store": True, "no-cache": True}, + metadata={ + "user_api_key": user_api_key, + "semantic-cache-embedding": True, + "trace_id": kwargs.get("metadata", {}).get("trace_id", None), + }, ) else: # convert to embedding + user_api_key = kwargs["litellm_params"]["metadata"].get("user_api_key", "") embedding_response = await litellm.aembedding( model=self.embedding_model, input=prompt, From e8e6fc6123dcd0acd568878f810cb05e7d69901f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 7 Feb 2024 19:24:27 -0800 Subject: [PATCH 3/4] (fix) remove extra statement --- litellm/caching.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/caching.py b/litellm/caching.py index 3522a9d436..f0ae7778af 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -495,7 +495,6 @@ class RedisSemanticCache(BaseCache): ) else: # convert to embedding - user_api_key = kwargs["litellm_params"]["metadata"].get("user_api_key", "") embedding_response = await litellm.aembedding( model=self.embedding_model, input=prompt, From 3837c77df9f0a5aad1933ea2bca3c8d8fcbdda9b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 7 Feb 2024 20:09:28 -0800 Subject: [PATCH 4/4] (feat) slack alerting when track callback fails --- litellm/proxy/proxy_server.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6c22efebc4..5008a5923a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -809,16 +809,15 @@ async def _PROXY_track_cost_callback( litellm_params = kwargs.get("litellm_params", {}) or {} proxy_server_request = litellm_params.get("proxy_server_request") or {} user_id = proxy_server_request.get("body", {}).get("user", None) + user_id = user_id or kwargs["litellm_params"]["metadata"].get( + "user_api_key_user_id", None + ) if kwargs.get("response_cost", None) is not None: response_cost = kwargs["response_cost"] user_api_key = kwargs["litellm_params"]["metadata"].get( "user_api_key", None ) - user_id = user_id or kwargs["litellm_params"]["metadata"].get( - "user_api_key_user_id", None - ) - if kwargs.get("cache_hit", False) == True: response_cost = 0.0 verbose_proxy_logger.info( @@ -853,7 +852,9 @@ async def _PROXY_track_cost_callback( ) except Exception as e: error_msg = f"error in tracking cost callback - {traceback.format_exc()}" - error_msg += f"\n Args to _PROXY_track_cost_callback\n kwargs: {kwargs}\n completion_response: {completion_response}" + model = kwargs.get("model", "") + metadata = kwargs.get("litellm_params", {}).get("metadata", {}) + error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n metadata: {metadata}\n" user_id = user_id or "not-found" asyncio.create_task( proxy_logging_obj.budget_alerts( @@ -864,9 +865,7 @@ async def _PROXY_track_cost_callback( error_message=error_msg, ) ) - verbose_proxy_logger.debug( - f"error in tracking cost callback - {traceback.format_exc()}" - ) + verbose_proxy_logger.debug(f"error in tracking cost callback - {error_msg}") async def update_database(