From 0f95a824c43079e4044c0d20857ac518bcff6a34 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 16:04:35 -0700 Subject: [PATCH 1/6] feat(prometheus_services.py): emit proxy latency for successful llm api requests uses prometheus histogram for this --- litellm/_service_logger.py | 60 ++++++++++++++++++--- litellm/caching.py | 26 ++++++--- litellm/integrations/prometheus_services.py | 2 + litellm/proxy/_new_secret_config.yaml | 12 ++--- litellm/proxy/utils.py | 3 ++ litellm/types/services.py | 4 +- 6 files changed, 87 insertions(+), 20 deletions(-) diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 814ec011a9..b2bd26c267 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -1,9 +1,12 @@ -import litellm +import litellm, traceback +from litellm.proxy._types import UserAPIKeyAuth from .types.services import ServiceTypes, ServiceLoggerPayload from .integrations.prometheus_services import PrometheusServicesLogger +from .integrations.custom_logger import CustomLogger +from datetime import timedelta -class ServiceLogging: +class ServiceLogging(CustomLogger): """ Separate class used for monitoring health of litellm-adjacent services (redis/postgres). """ @@ -14,7 +17,6 @@ class ServiceLogging: self.mock_testing_async_success_hook = 0 self.mock_testing_sync_failure_hook = 0 self.mock_testing_async_failure_hook = 0 - if "prometheus_system" in litellm.service_callback: self.prometheusServicesLogger = PrometheusServicesLogger() @@ -34,7 +36,9 @@ class ServiceLogging: if self.mock_testing: self.mock_testing_sync_failure_hook += 1 - async def async_service_success_hook(self, service: ServiceTypes, duration: float): + async def async_service_success_hook( + self, service: ServiceTypes, duration: float, call_type: str + ): """ - For counting if the redis, postgres call is successful """ @@ -42,7 +46,11 @@ class ServiceLogging: self.mock_testing_async_success_hook += 1 payload = ServiceLoggerPayload( - is_error=False, error=None, service=service, duration=duration + is_error=False, + error=None, + service=service, + duration=duration, + call_type=call_type, ) for callback in litellm.service_callback: if callback == "prometheus_system": @@ -51,7 +59,7 @@ class ServiceLogging: ) async def async_service_failure_hook( - self, service: ServiceTypes, duration: float, error: Exception + self, service: ServiceTypes, duration: float, error: Exception, call_type: str ): """ - For counting if the redis, postgres call is unsuccessful @@ -60,7 +68,11 @@ class ServiceLogging: self.mock_testing_async_failure_hook += 1 payload = ServiceLoggerPayload( - is_error=True, error=str(error), service=service, duration=duration + is_error=True, + error=str(error), + service=service, + duration=duration, + call_type=call_type, ) for callback in litellm.service_callback: if callback == "prometheus_system": @@ -69,3 +81,37 @@ class ServiceLogging: await self.prometheusServicesLogger.async_service_failure_hook( payload=payload ) + + async def async_post_call_failure_hook( + self, original_exception: Exception, user_api_key_dict: UserAPIKeyAuth + ): + """ + Hook to track failed litellm-service calls + """ + return await super().async_post_call_failure_hook( + original_exception, user_api_key_dict + ) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + """ + Hook to track latency for litellm proxy llm api calls + """ + try: + _duration = end_time - start_time + if isinstance(_duration, timedelta): + _duration = _duration.total_seconds() + elif isinstance(_duration, float): + pass + else: + raise Exception( + "Duration={} is not a float or timedelta object. type={}".format( + _duration, type(_duration) + ) + ) # invalid _duration value + await self.async_service_success_hook( + service=ServiceTypes.LITELLM, + duration=_duration, + call_type=kwargs["call_type"], + ) + except Exception as e: + raise e diff --git a/litellm/caching.py b/litellm/caching.py index c15813710a..79c816ff74 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -13,7 +13,6 @@ import json, traceback, ast, hashlib from typing import Optional, Literal, List, Union, Any, BinaryIO from openai._models import BaseModel as OpenAIObject from litellm._logging import verbose_logger -from litellm._service_logger import ServiceLogging from litellm.types.services import ServiceLoggerPayload, ServiceTypes import traceback @@ -132,6 +131,7 @@ class RedisCache(BaseCache): **kwargs, ): from ._redis import get_redis_client, get_redis_connection_pool + from litellm._service_logger import ServiceLogging import redis redis_kwargs = {} @@ -216,7 +216,9 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, duration=_duration + service=ServiceTypes.REDIS, + duration=_duration, + call_type="async_scan_iter", ) ) # DO NOT SLOW DOWN CALL B/C OF THIS return keys @@ -227,7 +229,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_scan_iter", ) ) raise e @@ -359,6 +364,7 @@ class RedisCache(BaseCache): self.service_logger_obj.async_service_success_hook( service=ServiceTypes.REDIS, duration=_duration, + call_type="async_increment", ) ) return result @@ -368,7 +374,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_increment", ) ) verbose_logger.error( @@ -497,7 +506,9 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, duration=_duration + service=ServiceTypes.REDIS, + duration=_duration, + call_type="async_batch_get_cache", ) ) @@ -519,7 +530,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_batch_get_cache", ) ) print_verbose(f"Error occurred in pipeline read - {str(e)}") diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 548d0a2a3a..4171593baf 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -30,6 +30,7 @@ class PrometheusServicesLogger: raise Exception( "Missing prometheus_client. Run `pip install prometheus-client`" ) + print("INITIALIZES PROMETHEUS SERVICE LOGGER!") self.Histogram = Histogram self.Counter = Counter @@ -151,6 +152,7 @@ class PrometheusServicesLogger: if self.mock_testing: self.mock_testing_success_calls += 1 + print(f"LOGS SUCCESSFUL CALL TO PROMETHEUS - payload={payload}") if payload.service.value in self.payload_to_prometheus_map: prom_objects = self.payload_to_prometheus_map[payload.service.value] for obj in prom_objects: diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 0f7c24576e..ca8b4c5393 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -31,12 +31,12 @@ litellm_settings: upperbound_key_generate_params: max_budget: os.environ/LITELLM_UPPERBOUND_KEYS_MAX_BUDGET -router_settings: - routing_strategy: usage-based-routing-v2 - redis_host: os.environ/REDIS_HOST - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT - enable_pre_call_checks: True +# router_settings: +# routing_strategy: usage-based-routing-v2 +# redis_host: os.environ/REDIS_HOST +# redis_password: os.environ/REDIS_PASSWORD +# redis_port: os.environ/REDIS_PORT +# enable_pre_call_checks: True general_settings: master_key: sk-1234 diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2fccbc74e8..6a34b54e15 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -18,6 +18,7 @@ from litellm.llms.custom_httpx.httpx_handler import HTTPHandler from litellm.proxy.hooks.parallel_request_limiter import ( _PROXY_MaxParallelRequestsHandler, ) +from litellm._service_logger import ServiceLogging from litellm import ModelResponse, EmbeddingResponse, ImageResponse from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter from litellm.proxy.hooks.tpm_rpm_limiter import _PROXY_MaxTPMRPMLimiter @@ -80,10 +81,12 @@ class ProxyLogging: def _init_litellm_callbacks(self): print_verbose(f"INITIALIZING LITELLM CALLBACKS!") + self.service_logging_obj = ServiceLogging() litellm.callbacks.append(self.max_parallel_request_limiter) litellm.callbacks.append(self.max_tpm_rpm_limiter) litellm.callbacks.append(self.max_budget_limiter) litellm.callbacks.append(self.cache_control_check) + litellm.callbacks.append(self.service_logging_obj) litellm.success_callback.append(self.response_taking_too_long_callback) for callback in litellm.callbacks: if callback not in litellm.input_callback: diff --git a/litellm/types/services.py b/litellm/types/services.py index ea5172ebc9..b694ca8078 100644 --- a/litellm/types/services.py +++ b/litellm/types/services.py @@ -5,11 +5,12 @@ from typing import Optional class ServiceTypes(enum.Enum): """ - Enum for litellm-adjacent services (redis/postgres/etc.) + Enum for litellm + litellm-adjacent services (redis/postgres/etc.) """ REDIS = "redis" DB = "postgres" + LITELLM = "self" class ServiceLoggerPayload(BaseModel): @@ -21,6 +22,7 @@ class ServiceLoggerPayload(BaseModel): error: Optional[str] = Field(None, description="what was the error") service: ServiceTypes = Field(description="who is this for? - postgres/redis") duration: float = Field(description="How long did the request take?") + call_type: str = Field(description="The call of the service, being made") def to_json(self, **kwargs): try: From 48d3b563d86331a9f25f909f2214bb2902dc25ca Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 16:06:12 -0700 Subject: [PATCH 2/6] docs(prometheus.md): add litellm service metrics to prometheus docs --- docs/my-website/docs/proxy/prometheus.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 7135639bfc..b1fb62ad51 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -14,6 +14,7 @@ model_list: model: gpt-3.5-turbo litellm_settings: success_callback: ["prometheus"] + failure_callback: ["prometheus"] ``` Start the proxy @@ -70,3 +71,4 @@ litellm_settings: |----------------------|--------------------------------------| | `litellm_redis_latency` | histogram latency for redis calls | | `litellm_redis_fails` | Number of failed redis calls | +| `litellm_self_latency` | Histogram latency for successful litellm api call | From d61250109e6dfbdeb810a7b9c0cfa557c02d1562 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 16:24:36 -0700 Subject: [PATCH 3/6] fix(proxy/utils.py): add call type and duration to proxy_logging failure calls this is for tracking failed db requests on prometheus --- litellm/proxy/utils.py | 156 ++++++++++++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 6a34b54e15..e45245909f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1,5 +1,5 @@ from typing import Optional, List, Any, Literal, Union -import os, subprocess, hashlib, importlib, asyncio, copy, json, aiohttp, httpx +import os, subprocess, hashlib, importlib, asyncio, copy, json, aiohttp, httpx, time import litellm, backoff from litellm.proxy._types import ( UserAPIKeyAuth, @@ -18,7 +18,7 @@ from litellm.llms.custom_httpx.httpx_handler import HTTPHandler from litellm.proxy.hooks.parallel_request_limiter import ( _PROXY_MaxParallelRequestsHandler, ) -from litellm._service_logger import ServiceLogging +from litellm._service_logger import ServiceLogging, ServiceTypes from litellm import ModelResponse, EmbeddingResponse, ImageResponse from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter from litellm.proxy.hooks.tpm_rpm_limiter import _PROXY_MaxTPMRPMLimiter @@ -458,7 +458,9 @@ class ProxyLogging: else: raise Exception("Missing SENTRY_DSN from environment") - async def failure_handler(self, original_exception, traceback_str=""): + async def failure_handler( + self, original_exception, duration: float, call_type: str, traceback_str="" + ): """ Log failed db read/writes @@ -483,6 +485,11 @@ class ProxyLogging: ) ) + if hasattr(self, "service_logging_obj"): + self.service_logging_obj.async_service_failure_hook( + service=ServiceTypes.DB, + ) + if litellm.utils.capture_exception: litellm.utils.capture_exception(error=original_exception) @@ -803,6 +810,7 @@ class PrismaClient: verbose_proxy_logger.debug( f"PrismaClient: get_generic_data: {key}, table_name: {table_name}" ) + start_time = time.time() try: if table_name == "users": response = await self.db.litellm_usertable.find_first( @@ -827,11 +835,17 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception get_generic_data: {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + traceback_str=error_traceback, + call_type="get_generic_data", ) ) + raise e @backoff.on_exception( @@ -869,6 +883,7 @@ class PrismaClient: ] = None, # pagination, number of rows to getch when find_all==True ): args_passed_in = locals() + start_time = time.time() verbose_proxy_logger.debug( f"PrismaClient: get_data - args_passed_in: {args_passed_in}" ) @@ -1122,9 +1137,15 @@ class PrismaClient: print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() verbose_proxy_logger.debug(error_traceback) + end_time = time.time() + _duration = end_time - start_time + asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="get_data", + traceback_str=error_traceback, ) ) raise e @@ -1147,6 +1168,7 @@ class PrismaClient: """ Add a key to the database. If it already exists, do nothing. """ + start_time = time.time() try: verbose_proxy_logger.debug("PrismaClient: insert_data: %s", data) if table_name == "key": @@ -1264,9 +1286,14 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception in insert_data: {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="insert_data", + traceback_str=error_traceback, ) ) raise e @@ -1297,6 +1324,7 @@ class PrismaClient: verbose_proxy_logger.debug( f"PrismaClient: update_data, table_name: {table_name}" ) + start_time = time.time() try: db_data = self.jsonify_object(data=data) if update_key_values is not None: @@ -1458,9 +1486,14 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception - update_data: {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_data", + traceback_str=error_traceback, ) ) raise e @@ -1485,6 +1518,7 @@ class PrismaClient: Ensure user owns that key, unless admin. """ + start_time = time.time() try: if tokens is not None and isinstance(tokens, List): hashed_tokens = [] @@ -1532,9 +1566,14 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception - delete_data: {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="delete_data", + traceback_str=error_traceback, ) ) raise e @@ -1548,6 +1587,7 @@ class PrismaClient: on_backoff=on_backoff, # specifying the function to call on backoff ) async def connect(self): + start_time = time.time() try: verbose_proxy_logger.debug( "PrismaClient: connect() called Attempting to Connect to DB" @@ -1563,9 +1603,14 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception connect(): {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="connect", + traceback_str=error_traceback, ) ) raise e @@ -1579,6 +1624,7 @@ class PrismaClient: on_backoff=on_backoff, # specifying the function to call on backoff ) async def disconnect(self): + start_time = time.time() try: await self.db.disconnect() except Exception as e: @@ -1587,9 +1633,14 @@ class PrismaClient: error_msg = f"LiteLLM Prisma Client Exception disconnect(): {str(e)}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( self.proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="disconnect", + traceback_str=error_traceback, ) ) raise e @@ -1598,16 +1649,35 @@ class PrismaClient: """ Health check endpoint for the prisma client """ - sql_query = """ - SELECT 1 - FROM "LiteLLM_VerificationToken" - LIMIT 1 - """ + start_time = time.time() + try: + sql_query = """ + SELECT 1 + FROM "LiteLLM_VerificationToken" + LIMIT 1 + """ - # Execute the raw query - # The asterisk before `user_id_list` unpacks the list into separate arguments - response = await self.db.query_raw(sql_query) - return response + # Execute the raw query + # The asterisk before `user_id_list` unpacks the list into separate arguments + response = await self.db.query_raw(sql_query) + return response + except Exception as e: + import traceback + + error_msg = f"LiteLLM Prisma Client Exception disconnect(): {str(e)}" + print_verbose(error_msg) + error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time + asyncio.create_task( + self.proxy_logging_obj.failure_handler( + original_exception=e, + duration=_duration, + call_type="health_check", + traceback_str=error_traceback, + ) + ) + raise e class DBClient: @@ -1983,6 +2053,7 @@ async def update_spend( ### UPDATE USER TABLE ### if len(prisma_client.user_list_transactons.keys()) > 0: for i in range(n_retry_times + 1): + start_time = time.time() try: async with prisma_client.db.tx( timeout=timedelta(seconds=60) @@ -2013,9 +2084,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e @@ -2023,6 +2099,7 @@ async def update_spend( ### UPDATE END-USER TABLE ### if len(prisma_client.end_user_list_transactons.keys()) > 0: for i in range(n_retry_times + 1): + start_time = time.time() try: async with prisma_client.db.tx( timeout=timedelta(seconds=60) @@ -2059,9 +2136,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e @@ -2069,6 +2151,7 @@ async def update_spend( ### UPDATE KEY TABLE ### if len(prisma_client.key_list_transactons.keys()) > 0: for i in range(n_retry_times + 1): + start_time = time.time() try: async with prisma_client.db.tx( timeout=timedelta(seconds=60) @@ -2099,9 +2182,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e @@ -2114,6 +2202,7 @@ async def update_spend( ) if len(prisma_client.team_list_transactons.keys()) > 0: for i in range(n_retry_times + 1): + start_time = time.time() try: async with prisma_client.db.tx( timeout=timedelta(seconds=60) @@ -2149,9 +2238,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e @@ -2159,6 +2253,7 @@ async def update_spend( ### UPDATE ORG TABLE ### if len(prisma_client.org_list_transactons.keys()) > 0: for i in range(n_retry_times + 1): + start_time = time.time() try: async with prisma_client.db.tx( timeout=timedelta(seconds=60) @@ -2189,9 +2284,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e @@ -2206,6 +2306,7 @@ async def update_spend( if len(prisma_client.spend_log_transactions) > 0: for _ in range(n_retry_times + 1): + start_time = time.time() try: base_url = os.getenv("SPEND_LOGS_URL", None) ## WRITE TO SEPARATE SERVER ## @@ -2271,9 +2372,14 @@ async def update_spend( ) print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() + end_time = time.time() + _duration = end_time - start_time asyncio.create_task( proxy_logging_obj.failure_handler( - original_exception=e, traceback_str=error_traceback + original_exception=e, + duration=_duration, + call_type="update_spend", + traceback_str=error_traceback, ) ) raise e From 919a2876f14d804c0db868df1ff5feba444ceb38 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 16:30:29 -0700 Subject: [PATCH 4/6] fix(proxy/utils.py): add prometheus failed db request tracking --- litellm/_service_logger.py | 15 +++++++++++++-- litellm/proxy/utils.py | 3 +++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index b2bd26c267..0c6996b102 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -4,6 +4,7 @@ from .types.services import ServiceTypes, ServiceLoggerPayload from .integrations.prometheus_services import PrometheusServicesLogger from .integrations.custom_logger import CustomLogger from datetime import timedelta +from typing import Union class ServiceLogging(CustomLogger): @@ -59,7 +60,11 @@ class ServiceLogging(CustomLogger): ) async def async_service_failure_hook( - self, service: ServiceTypes, duration: float, error: Exception, call_type: str + self, + service: ServiceTypes, + duration: float, + error: Union[str, Exception], + call_type: str, ): """ - For counting if the redis, postgres call is unsuccessful @@ -67,9 +72,15 @@ class ServiceLogging(CustomLogger): if self.mock_testing: self.mock_testing_async_failure_hook += 1 + error_message = "" + if isinstance(error, Exception): + error_message = str(error) + elif isinstance(error, str): + error_message = error + payload = ServiceLoggerPayload( is_error=True, - error=str(error), + error=error_message, service=service, duration=duration, call_type=call_type, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e45245909f..8572fd70c0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -488,6 +488,9 @@ class ProxyLogging: if hasattr(self, "service_logging_obj"): self.service_logging_obj.async_service_failure_hook( service=ServiceTypes.DB, + duration=duration, + error=error_message, + call_type=call_type, ) if litellm.utils.capture_exception: From 0afb5d6573dbf4af91ad371abee850330f9635eb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 16:53:34 -0700 Subject: [PATCH 5/6] fix(caching.py): add call type to service logger object --- litellm/caching.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 79c816ff74..99f9012b1b 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -272,7 +272,9 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, duration=_duration + service=ServiceTypes.REDIS, + duration=_duration, + call_type="async_set_cache", ) ) except Exception as e: @@ -280,7 +282,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_set_cache", ) ) # NON blocking - notify users Redis is throwing an exception @@ -321,7 +326,9 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, duration=_duration + service=ServiceTypes.REDIS, + duration=_duration, + call_type="async_set_cache_pipeline", ) ) return results @@ -331,7 +338,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_set_cache_pipeline", ) ) From f65005f273c46133295429b3447d69035d26362e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 18 Apr 2024 17:21:10 -0700 Subject: [PATCH 6/6] fix(caching.py): fix error logging --- litellm/caching.py | 9 +++++++-- litellm/tests/test_custom_logger.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 99f9012b1b..bf1d61eec3 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -478,7 +478,9 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, duration=_duration + service=ServiceTypes.REDIS, + duration=_duration, + call_type="async_get_cache", ) ) return response @@ -488,7 +490,10 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + call_type="async_get_cache", ) ) # NON blocking - notify users Redis is throwing an exception diff --git a/litellm/tests/test_custom_logger.py b/litellm/tests/test_custom_logger.py index 0b85b463ce..347fb57ba6 100644 --- a/litellm/tests/test_custom_logger.py +++ b/litellm/tests/test_custom_logger.py @@ -412,7 +412,7 @@ async def test_cost_tracking_with_caching(): """ from litellm import Cache - litellm.set_verbose = False + litellm.set_verbose = True litellm.cache = Cache( type="redis", host=os.environ["REDIS_HOST"],