diff --git a/litellm/caching.py b/litellm/caching.py index 6bf53ea451..f996a58735 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -531,6 +531,9 @@ class RedisSemanticCache(BaseCache): return None pass + async def _index_info(self): + return await self.index.ainfo() + class S3Cache(BaseCache): def __init__( diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 3c3e793dfb..3031868ec7 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -255,6 +255,7 @@ class LangFuseLogger: if key in [ "user_api_key", "user_api_key_user_id", + "semantic-similarity", ]: tags.append(f"{key}:{value}") if "cache_hit" in kwargs: diff --git a/litellm/main.py b/litellm/main.py index 384dadc32d..b18221607f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -10,7 +10,6 @@ import os, openai, sys, json, inspect, uuid, datetime, threading from typing import Any, Literal, Union from functools import partial - import dotenv, traceback, random, asyncio, time, contextvars from copy import deepcopy import httpx diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 326544f41e..a8144e9d48 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -78,7 +78,9 @@ litellm_settings: type: "redis-semantic" similarity_threshold: 0.8 redis_semantic_cache_embedding_model: azure-embedding-model - # cache: True + upperbound_key_generate_params: + max_budget: 100 + duration: "30d" # setting callback class # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 30233fc137..945bb5605f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4076,19 +4076,29 @@ async def health_readiness(): cache_type = None if litellm.cache is not None: + from litellm.caching import RedisSemanticCache + cache_type = litellm.cache.type - if prisma_client is not None: # if db passed in, check if it's connected - if prisma_client.db.is_connected() == True: - response_object = {"db": "connected"} + if isinstance(litellm.cache.cache, RedisSemanticCache): + # ping the cache + try: + index_info = await litellm.cache.cache._index_info() + except Exception as e: + index_info = "index does not exist - error: " + str(e) + cache_type = {"type": cache_type, "index_info": index_info} - return { - "status": "healthy", - "db": "connected", - "cache": cache_type, - "litellm_version": version, - "success_callbacks": litellm.success_callback, - } + if prisma_client is not None: # if db passed in, check if it's connected + await prisma_client.health_check() # test the db connection + response_object = {"db": "connected"} + + return { + "status": "healthy", + "db": "connected", + "cache": cache_type, + "litellm_version": version, + "success_callbacks": litellm.success_callback, + } else: return { "status": "healthy", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 84b09d7265..5c5b5b7727 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -472,8 +472,6 @@ class PrismaClient: reset_at: Optional[datetime] = None, ): try: - print_verbose("PrismaClient: get_data") - response: Any = None if token is not None or (table_name is not None and table_name == "key"): # check if plain text or hash @@ -885,6 +883,21 @@ class PrismaClient: ) raise e + async def health_check(self): + """ + Health check endpoint for the prisma client + """ + sql_query = """ + SELECT 1 + FROM "LiteLLM_VerificationToken" + LIMIT 1 + """ + + # Execute the raw query + # The asterisk before `user_id_list` unpacks the list into separate arguments + response = await self.db.query_raw(sql_query) + return response + class DBClient: """ diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index b075e48190..80a4372a57 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -41,7 +41,7 @@ def test_completion_custom_provider_model_name(): messages=messages, logger_fn=logger_fn, ) - # Add any assertions here to check the, response + # Add any assertions here to check the,response print(response) print(response["choices"][0]["finish_reason"]) except litellm.Timeout as e: diff --git a/requirements.txt b/requirements.txt index 3ace5872ad..55c5f14568 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,7 @@ pyyaml>=6.0.1 # server dep uvicorn==0.22.0 # server dep gunicorn==21.2.0 # server dep boto3==1.28.58 # aws bedrock/sagemaker calls -redis==4.6.0 # caching -redisvl==0.0.7 # semantic caching +redis==5.0.0 # caching numpy==1.24.3 # semantic caching prisma==0.11.0 # for db mangum==0.17.0 # for aws lambda functions