Merge pull request #5324 from BerriAI/litellm_add_qdrant_litellm_proxy

[Feat-Proxy] Add Qdrant Semantic Caching Support

docs/my-website/docs/caching/all_caches.md

@@ -161,8 +161,7 @@ random_number = random.randint(
 print("testing semantic caching")
 litellm.cache = Cache(
     type="qdrant-semantic",
-    qdrant_host_type="cloud", # can be either 'cloud' or 'local'
-    qdrant_url=os.environ["QDRANT_URL"], 
+    qdrant_api_base=os.environ["QDRANT_API_BASE"], 
     qdrant_api_key=os.environ["QDRANT_API_KEY"],
     qdrant_collection_name="your_collection_name", # any name of your collection
     similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
@@ -491,12 +490,11 @@ def __init__(
     disk_cache_dir=None,
 
     # qdrant cache params
-    qdrant_url: Optional[str] = None,
+    qdrant_api_base: Optional[str] = None,
     qdrant_api_key: Optional[str] = None,
     qdrant_collection_name: Optional[str] = None,
     qdrant_quantization_config: Optional[str] = None,
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
-    qdrant_host_type: Optional[Literal["local","cloud"]] = "local",
 
     **kwargs
 ):

docs/my-website/docs/proxy/caching.md

@@ -7,6 +7,7 @@ Cache LLM Responses
 LiteLLM supports:
 - In Memory Cache
 - Redis Cache 
+- Qdrant Semantic Cache
 - Redis Semantic Cache
 - s3 Bucket Cache 
 
@@ -103,6 +104,66 @@ $ litellm --config /path/to/config.yaml
 ```
 </TabItem>
 
+
+<TabItem value="qdrant-semantic" label="Qdrant Semantic cache">
+
+Caching can be enabled by adding the `cache` key in the `config.yaml`
+
+#### Step 1: Add `cache` to the config.yaml
+```yaml
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  - model_name: openai-embedding
+    litellm_params:
+      model: openai/text-embedding-3-small
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  set_verbose: True
+  cache: True          # set cache responses to True, litellm defaults to using a redis cache
+  cache_params:
+    type: qdrant-semantic
+    qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
+    qdrant_collection_name: test_collection
+    qdrant_quantization_config: binary
+    similarity_threshold: 0.8   # similarity threshold for semantic cache
+```
+
+#### Step 2: Add Qdrant Credentials to your .env
+
+```shell
+QDRANT_API_KEY = "16rJUMBRx*************"
+QDRANT_API_BASE = "https://5392d382-45*********.cloud.qdrant.io"
+```
+
+#### Step 3: Run proxy with config
+```shell
+$ litellm --config /path/to/config.yaml
+```
+
+
+#### Step 4. Test it
+
+```shell
+curl -i http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "fake-openai-endpoint",
+    "messages": [
+      {"role": "user", "content": "Hello"}
+    ]
+  }'
+```
+
+**Expect to see `x-litellm-semantic-similarity` in the response headers when semantic caching is one**
+
+</TabItem>
+
 <TabItem value="s3" label="s3 cache">
 
 #### Step 1: Add `cache` to the config.yaml
@@ -182,6 +243,9 @@ REDIS_<redis-kwarg-name> = ""
 $ litellm --config /path/to/config.yaml
 ```
 </TabItem>
+
+
+
 </Tabs>
 
 

docs/my-website/sidebars.js

@@ -74,6 +74,7 @@ const sidebars = {
         "proxy/alerting",
         "proxy/ui",
         "proxy/prometheus",
+        "proxy/caching",
         "proxy/pass_through",
         "proxy/email",
         "proxy/multiple_admins",
@@ -88,7 +89,6 @@ const sidebars = {
         "proxy/health",
         "proxy/debugging",
         "proxy/pii_masking",
-        "proxy/caching",
         "proxy/call_hooks",
         "proxy/rules",
         "proxy/cli", 

litellm/caching.py

@@ -1223,7 +1223,7 @@ class RedisSemanticCache(BaseCache):
 class QdrantSemanticCache(BaseCache):
     def __init__(
         self,
-        qdrant_url=None,
+        qdrant_api_base=None,
         qdrant_api_key=None,
         collection_name=None,
         similarity_threshold=None,
@@ -1251,18 +1251,31 @@ class QdrantSemanticCache(BaseCache):
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
         headers = {}
-        if qdrant_url is None:
-            qdrant_url = os.getenv("QDRANT_URL")
-        if qdrant_api_key is None:
-            qdrant_api_key = os.getenv("QDRANT_API_KEY")
-        if qdrant_url is not None and qdrant_api_key is not None:
-            headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"}
-        else:
-            raise Exception("Qdrant url and api_key must be")
 
-        self.qdrant_url = qdrant_url
+        # check if defined as os.environ/ variable
+        if qdrant_api_base:
+            if isinstance(qdrant_api_base, str) and qdrant_api_base.startswith(
+                "os.environ/"
+            ):
+                qdrant_api_base = litellm.get_secret(qdrant_api_base)
+        if qdrant_api_key:
+            if isinstance(qdrant_api_key, str) and qdrant_api_key.startswith(
+                "os.environ/"
+            ):
+                qdrant_api_key = litellm.get_secret(qdrant_api_key)
+
+        qdrant_api_base = (
+            qdrant_api_base or os.getenv("QDRANT_URL") or os.getenv("QDRANT_API_BASE")
+        )
+        qdrant_api_key = qdrant_api_key or os.getenv("QDRANT_API_KEY")
+        headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"}
+
+        if qdrant_api_key is None or qdrant_api_base is None:
+            raise ValueError("Qdrant url and api_key must be")
+
+        self.qdrant_api_base = qdrant_api_base
         self.qdrant_api_key = qdrant_api_key
-        print_verbose(f"qdrant semantic-cache qdrant_url: {self.qdrant_url}")
+        print_verbose(f"qdrant semantic-cache qdrant_api_base: {self.qdrant_api_base}")
 
         self.headers = headers
 
@@ -1274,7 +1287,7 @@ class QdrantSemanticCache(BaseCache):
                 "Quantization config is not provided. Default binary quantization will be used."
             )
         collection_exists = self.sync_client.get(
-            url=f"{self.qdrant_url}/collections/{self.collection_name}/exists",
+            url=f"{self.qdrant_api_base}/collections/{self.collection_name}/exists",
             headers=self.headers,
         )
         if collection_exists.status_code != 200:
@@ -1284,7 +1297,7 @@ class QdrantSemanticCache(BaseCache):
 
         if collection_exists.json()["result"]["exists"]:
             collection_details = self.sync_client.get(
-                url=f"{self.qdrant_url}/collections/{self.collection_name}",
+                url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
                 headers=self.headers,
             )
             self.collection_info = collection_details.json()
@@ -1312,7 +1325,7 @@ class QdrantSemanticCache(BaseCache):
                 )
 
             new_collection_status = self.sync_client.put(
-                url=f"{self.qdrant_url}/collections/{self.collection_name}",
+                url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
                 json={
                     "vectors": {"size": 1536, "distance": "Cosine"},
                     "quantization_config": quantization_params,
@@ -1321,7 +1334,7 @@ class QdrantSemanticCache(BaseCache):
             )
             if new_collection_status.json()["result"]:
                 collection_details = self.sync_client.get(
-                    url=f"{self.qdrant_url}/collections/{self.collection_name}",
+                    url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
                     headers=self.headers,
                 )
                 self.collection_info = collection_details.json()
@@ -1378,7 +1391,7 @@ class QdrantSemanticCache(BaseCache):
             ]
         }
         keys = self.sync_client.put(
-            url=f"{self.qdrant_url}/collections/{self.collection_name}/points",
+            url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points",
             headers=self.headers,
             json=data,
         )
@@ -1417,7 +1430,7 @@ class QdrantSemanticCache(BaseCache):
         }
 
         search_response = self.sync_client.post(
-            url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search",
+            url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
             headers=self.headers,
             json=data,
         )
@@ -1506,7 +1519,7 @@ class QdrantSemanticCache(BaseCache):
         }
 
         keys = await self.async_client.put(
-            url=f"{self.qdrant_url}/collections/{self.collection_name}/points",
+            url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points",
             headers=self.headers,
             json=data,
         )
@@ -1564,7 +1577,7 @@ class QdrantSemanticCache(BaseCache):
         }
 
         search_response = await self.async_client.post(
-            url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search",
+            url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
             headers=self.headers,
             json=data,
         )
@@ -2111,12 +2124,11 @@ class Cache:
         redis_semantic_cache_embedding_model="text-embedding-ada-002",
         redis_flush_size=None,
         disk_cache_dir=None,
-        qdrant_url: Optional[str] = None,
+        qdrant_api_base: Optional[str] = None,
         qdrant_api_key: Optional[str] = None,
         qdrant_collection_name: Optional[str] = None,
         qdrant_quantization_config: Optional[str] = None,
         qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
-        qdrant_host_type: Optional[Literal["local", "cloud"]] = "local",
         **kwargs,
     ):
         """
@@ -2127,9 +2139,8 @@ class Cache:
             host (str, optional): The host address for the Redis cache. Required if type is "redis".
             port (int, optional): The port number for the Redis cache. Required if type is "redis".
             password (str, optional): The password for the Redis cache. Required if type is "redis".
-            qdrant_url (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic".
-            qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster. Required if qdrant_host_type is "cloud" and optional if qdrant_host_type is "local".
-            qdrant_host_type (str, optional): Can be either "local" or "cloud". Should be "local" when you are running a local qdrant cluster or "cloud" when you are using a qdrant cloud cluster.
+            qdrant_api_base (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic".
+            qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster.
             qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
 
@@ -2158,13 +2169,12 @@ class Cache:
             )
         elif type == "qdrant-semantic":
             self.cache = QdrantSemanticCache(
-                qdrant_url=qdrant_url,
+                qdrant_api_base=qdrant_api_base,
                 qdrant_api_key=qdrant_api_key,
                 collection_name=qdrant_collection_name,
                 similarity_threshold=similarity_threshold,
                 quantization_config=qdrant_quantization_config,
                 embedding_model=qdrant_semantic_cache_embedding_model,
-                host_type=qdrant_host_type,
             )
         elif type == "local":
             self.cache = InMemoryCache()

litellm/proxy/common_utils/callback_utils.py

@@ -285,14 +285,18 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str,
     return headers
 
 
-def get_applied_guardrails_header(request_data: Dict) -> Optional[Dict]:
+def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]:
     _metadata = request_data.get("metadata", None) or {}
+    headers = {}
     if "applied_guardrails" in _metadata:
-        return {
-            "x-litellm-applied-guardrails": ",".join(_metadata["applied_guardrails"]),
-        }
+        headers["x-litellm-applied-guardrails"] = ",".join(
+            _metadata["applied_guardrails"]
+        )
 
-    return None
+    if "semantic-similarity" in _metadata:
+        headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"])
+
+    return headers
 
 
 def add_guardrail_to_applied_guardrails_header(

litellm/proxy/proxy_config.yaml

@@ -4,15 +4,17 @@ model_list:
       model: openai/fake
       api_key: fake-key
       api_base: https://exampleopenaiendpoint-production.up.railway.app/
-
-guardrails:
-  - guardrail_name: "lakera-pre-guard"
+  - model_name: openai-embedding
     litellm_params:
-      guardrail: lakera  # supported values: "aporia", "bedrock", "lakera"
-      mode: "during_call"
-      api_key: os.environ/LAKERA_API_KEY
-      api_base: os.environ/LAKERA_API_BASE
-      category_thresholds:
-        prompt_injection: 0.1
-        jailbreak: 0.1
-  
+      model: openai/text-embedding-3-small
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  set_verbose: True
+  cache: True          # set cache responses to True, litellm defaults to using a redis cache
+  cache_params:
+    type: qdrant-semantic
+    qdrant_semantic_cache_embedding_model: openai-embedding
+    qdrant_collection_name: test_collection
+    qdrant_quantization_config: binary
+    similarity_threshold: 0.8   # similarity threshold for semantic cache

litellm/proxy/proxy_server.py

@@ -149,7 +149,7 @@ from litellm.proxy.common_utils.admin_ui_utils import (
     show_missing_vars_in_env,
 )
 from litellm.proxy.common_utils.callback_utils import (
-    get_applied_guardrails_header,
+    get_logging_caching_headers,
     get_remaining_tokens_and_requests_from_request_data,
     initialize_callbacks_on_proxy,
 )
@@ -543,9 +543,9 @@ def get_custom_headers(
         )
         headers.update(remaining_tokens_header)
 
-        applied_guardrails = get_applied_guardrails_header(request_data)
-        if applied_guardrails:
-            headers.update(applied_guardrails)
+        logging_caching_headers = get_logging_caching_headers(request_data)
+        if logging_caching_headers:
+            headers.update(logging_caching_headers)
 
     try:
         return {

litellm/tests/test_caching.py

@@ -1746,7 +1746,7 @@ async def test_qdrant_semantic_cache_acompletion():
     litellm.cache = Cache(
         type="qdrant-semantic",
         _host_type="cloud",
-        qdrant_url=os.getenv("QDRANT_URL"),
+        qdrant_api_base=os.getenv("QDRANT_URL"),
         qdrant_api_key=os.getenv("QDRANT_API_KEY"),
         qdrant_collection_name="test_collection",
         similarity_threshold=0.8,
@@ -1794,8 +1794,7 @@ async def test_qdrant_semantic_cache_acompletion_stream():
         ]
         litellm.cache = Cache(
             type="qdrant-semantic",
-            qdrant_host_type="cloud",
-            qdrant_url=os.getenv("QDRANT_URL"),
+            qdrant_api_base=os.getenv("QDRANT_URL"),
             qdrant_api_key=os.getenv("QDRANT_API_KEY"),
             qdrant_collection_name="test_collection",
             similarity_threshold=0.8,