From 7bda5c7cac56a206234016ba7f01317253b53ce6 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 10:59:49 -0700 Subject: [PATCH 01/12] chore(caching): isolate semantic cache entries --- litellm/caching/qdrant_semantic_cache.py | 151 +++++++------ litellm/caching/redis_semantic_cache.py | 116 ++++++++-- .../caching/test_qdrant_semantic_cache.py | 113 ++++++++-- .../caching/test_redis_semantic_cache.py | 207 +++++++++++++++++- 4 files changed, 468 insertions(+), 119 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 5e3713e5a1..b856423cbb 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -16,12 +16,17 @@ from typing import Any, cast import litellm from litellm._logging import print_verbose from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_str_from_messages, +) from litellm.types.utils import EmbeddingResponse from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): + CACHE_KEY_FIELD_NAME = "litellm_cache_key" + def __init__( # noqa: PLR0915 self, qdrant_api_base=None, @@ -170,15 +175,66 @@ class QdrantSemanticCache(BaseCache): cached_response = ast.literal_eval(cached_response) return cached_response + def _get_qdrant_cache_key_filter(self, key: str) -> dict: + return { + "must": [ + { + "key": self.CACHE_KEY_FIELD_NAME, + "match": {"value": str(key)}, + } + ] + } + + def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: + cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) + return cached_key is not None and str(cached_key) == str(key) + + async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: + llm_model_list = None + llm_router = None + + try: + from litellm.proxy.proxy_server import ( + llm_model_list as proxy_llm_model_list, + llm_router as proxy_llm_router, + ) + + llm_model_list = proxy_llm_model_list + llm_router = proxy_llm_router + except ImportError: + pass + + router_model_names = ( + [m["model_name"] for m in llm_model_list] + if llm_model_list is not None + else [] + ) + if llm_router is not None and self.embedding_model in router_model_names: + user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") + return await llm_router.aembedding( + model=self.embedding_model, + input=prompt, + cache={"no-store": True, "no-cache": True}, + metadata={ + "user_api_key": user_api_key, + "semantic-cache-embedding": True, + "trace_id": kwargs.get("metadata", {}).get("trace_id", None), + }, + ) + + return await litellm.aembedding( + model=self.embedding_model, + input=prompt, + cache={"no-store": True, "no-cache": True}, + ) + def set_cache(self, key, value, **kwargs): print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}") from litellm._uuid import uuid # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) # create an embedding for prompt embedding_response = cast( @@ -202,6 +258,7 @@ class QdrantSemanticCache(BaseCache): "id": str(uuid.uuid4()), "vector": embedding, "payload": { + self.CACHE_KEY_FIELD_NAME: str(key), "text": prompt, "response": value, }, @@ -220,9 +277,7 @@ class QdrantSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) # convert to embedding embedding_response = cast( @@ -248,6 +303,7 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, + "filter": self._get_qdrant_cache_key_filter(key), } search_response = self.sync_client.post( @@ -264,7 +320,12 @@ class QdrantSemanticCache(BaseCache): return None similarity = results[0]["score"] - cached_prompt = results[0]["payload"]["text"] + payload = results[0]["payload"] + if not self._payload_matches_cache_key(payload=payload, key=key): + print_verbose("Qdrant semantic-cache hit did not match cache key scope") + return None + + cached_prompt = payload["text"] # check similarity, if more than self.similarity_threshold, return results print_verbose( @@ -272,7 +333,7 @@ class QdrantSemanticCache(BaseCache): ) if similarity >= self.similarity_threshold: # cache hit ! - cached_value = results[0]["payload"]["response"] + cached_value = payload["response"] print_verbose( f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}" ) @@ -285,40 +346,12 @@ class QdrantSemanticCache(BaseCache): async def async_set_cache(self, key, value, **kwargs): from litellm._uuid import uuid - from litellm.proxy.proxy_server import llm_model_list, llm_router - print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}") # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] - # create an embedding for prompt - router_model_names = ( - [m["model_name"] for m in llm_model_list] - if llm_model_list is not None - else [] - ) - if llm_router is not None and self.embedding_model in router_model_names: - user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") - embedding_response = await llm_router.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - metadata={ - "user_api_key": user_api_key, - "semantic-cache-embedding": True, - "trace_id": kwargs.get("metadata", {}).get("trace_id", None), - }, - ) - else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + prompt = get_str_from_messages(messages) + embedding_response = await self._get_async_embedding(prompt, **kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] @@ -332,6 +365,7 @@ class QdrantSemanticCache(BaseCache): "id": str(uuid.uuid4()), "vector": embedding, "payload": { + self.CACHE_KEY_FIELD_NAME: str(key), "text": prompt, "response": value, }, @@ -348,38 +382,12 @@ class QdrantSemanticCache(BaseCache): async def async_get_cache(self, key, **kwargs): print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}") - from litellm.proxy.proxy_server import llm_model_list, llm_router # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) - router_model_names = ( - [m["model_name"] for m in llm_model_list] - if llm_model_list is not None - else [] - ) - if llm_router is not None and self.embedding_model in router_model_names: - user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") - embedding_response = await llm_router.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - metadata={ - "user_api_key": user_api_key, - "semantic-cache-embedding": True, - "trace_id": kwargs.get("metadata", {}).get("trace_id", None), - }, - ) - else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + embedding_response = await self._get_async_embedding(prompt, **kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] @@ -395,6 +403,7 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, + "filter": self._get_qdrant_cache_key_filter(key), } search_response = await self.async_client.post( @@ -414,7 +423,13 @@ class QdrantSemanticCache(BaseCache): return None similarity = results[0]["score"] - cached_prompt = results[0]["payload"]["text"] + payload = results[0]["payload"] + if not self._payload_matches_cache_key(payload=payload, key=key): + print_verbose("Qdrant semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 + return None + + cached_prompt = payload["text"] # check similarity, if more than self.similarity_threshold, return results print_verbose( @@ -426,7 +441,7 @@ class QdrantSemanticCache(BaseCache): if similarity >= self.similarity_threshold: # cache hit ! - cached_value = results[0]["payload"]["response"] + cached_value = payload["response"] print_verbose( f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}" ) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index c76f27377d..06e9b4d0fa 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -35,6 +35,11 @@ class RedisSemanticCache(BaseCache): """ DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" + CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" + CACHE_KEY_FILTERABLE_FIELD: Dict[str, str] = { + "name": CACHE_KEY_FIELD_NAME, + "type": "tag", + } def __init__( self, @@ -66,8 +71,8 @@ class RedisSemanticCache(BaseCache): Exception: If similarity_threshold is not provided or required Redis connection information is missing """ - from redisvl.extensions.llmcache import SemanticCache - from redisvl.utils.vectorize import CustomTextVectorizer + from redisvl.extensions.llmcache import SemanticCache # type: ignore[import-not-found, import-untyped] + from redisvl.utils.vectorize import CustomTextVectorizer # type: ignore[import-not-found, import-untyped] if index_name is None: index_name = self.DEFAULT_REDIS_INDEX_NAME @@ -109,14 +114,61 @@ class RedisSemanticCache(BaseCache): # Initialize the Redis vectorizer and cache cache_vectorizer = CustomTextVectorizer(self._get_embedding) - self.llmcache = SemanticCache( - name=index_name, + self.llmcache = self._init_semantic_cache( + semantic_cache_cls=SemanticCache, + index_name=index_name, redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - overwrite=False, + cache_vectorizer=cache_vectorizer, ) + def _init_semantic_cache( + self, + semantic_cache_cls: Any, + index_name: str, + redis_url: str, + cache_vectorizer: Any, + ) -> Any: + try: + return semantic_cache_cls( + name=index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + except ValueError as exc: + if "schema does not match" not in str(exc): + raise + + isolated_index_name = f"{index_name}_isolated" + print_verbose( + "Redis semantic-cache existing index schema is not isolated; " + f"using isolated index - {isolated_index_name}" + ) + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + + def _get_cache_filters(self, key: str) -> Dict[str, str]: + return {self.CACHE_KEY_FIELD_NAME: str(key)} + + def _get_cache_key_filter_expression(self, key: str) -> Any: + from redisvl.query.filter import Tag # type: ignore[import-not-found, import-untyped] + + return Tag(self.CACHE_KEY_FIELD_NAME) == str(key) + + def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool: + cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) + if isinstance(cached_key, bytes): + cached_key = cached_key.decode("utf-8") + return cached_key is not None and str(cached_key) == str(key) + def _get_ttl(self, **kwargs) -> Optional[int]: """ Get the TTL (time-to-live) value for cache entries. @@ -188,7 +240,7 @@ class RedisSemanticCache(BaseCache): Store a value in the semantic cache. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live @@ -206,12 +258,15 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) + store_kwargs: Dict[str, Any] = {"filters": self._get_cache_filters(key)} + # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) if ttl is not None: - self.llmcache.store(prompt, value_str, ttl=int(ttl)) + store_kwargs["ttl"] = int(ttl) + self.llmcache.store(prompt, value_str, **store_kwargs) else: - self.llmcache.store(prompt, value_str) + self.llmcache.store(prompt, value_str, **store_kwargs) except Exception as e: print_verbose( f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}" @@ -222,7 +277,7 @@ class RedisSemanticCache(BaseCache): Retrieve a semantically similar cached response. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: @@ -238,8 +293,12 @@ class RedisSemanticCache(BaseCache): return None prompt = get_str_from_messages(messages) - # Check the cache for semantically similar prompts - results = self.llmcache.check(prompt=prompt) + # Check the cache for semantically similar prompts in this exact + # LiteLLM cache-key scope. + results = self.llmcache.check( + prompt=prompt, + filter_expression=self._get_cache_key_filter_expression(key), + ) # Return None if no similar prompts found if not results: @@ -247,6 +306,9 @@ class RedisSemanticCache(BaseCache): # Process the best matching result cache_hit = results[0] + if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): + print_verbose("Redis semantic-cache hit did not match cache key scope") + return None vector_distance = float(cache_hit["vector_distance"]) # Convert vector distance back to similarity score @@ -321,7 +383,7 @@ class RedisSemanticCache(BaseCache): Asynchronously store a value in the semantic cache. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live @@ -341,20 +403,25 @@ class RedisSemanticCache(BaseCache): # Generate embedding for the value (response) to cache prompt_embedding = await self._get_async_embedding(prompt, **kwargs) + store_kwargs: Dict[str, Any] = { + "vector": prompt_embedding, + "filters": self._get_cache_filters(key), + } + # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) if ttl is not None: + store_kwargs["ttl"] = ttl await self.llmcache.astore( prompt, value_str, - vector=prompt_embedding, # Pass through custom embedding - ttl=ttl, + **store_kwargs, ) else: await self.llmcache.astore( prompt, value_str, - vector=prompt_embedding, # Pass through custom embedding + **store_kwargs, ) except Exception as e: print_verbose(f"Error in async_set_cache: {str(e)}") @@ -364,7 +431,7 @@ class RedisSemanticCache(BaseCache): Asynchronously retrieve a semantically similar cached response. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: @@ -385,8 +452,13 @@ class RedisSemanticCache(BaseCache): # Generate embedding for the prompt prompt_embedding = await self._get_async_embedding(prompt, **kwargs) - # Check the cache for semantically similar prompts - results = await self.llmcache.acheck(prompt=prompt, vector=prompt_embedding) + # Check the cache for semantically similar prompts in this exact + # LiteLLM cache-key scope. + results = await self.llmcache.acheck( + prompt=prompt, + vector=prompt_embedding, + filter_expression=self._get_cache_key_filter_expression(key), + ) # handle results / cache hit if not results: @@ -396,6 +468,10 @@ class RedisSemanticCache(BaseCache): return None cache_hit = results[0] + if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): + print_verbose("Redis semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 + return None vector_distance = float(cache_hit["vector_distance"]) # Convert vector distance back to similarity diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 13dc4b5812..f357d97341 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -19,9 +19,7 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -67,9 +65,7 @@ def test_qdrant_semantic_cache_get_cache_hit(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -98,6 +94,7 @@ def test_qdrant_semantic_cache_get_cache_hit(): "result": [ { "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", "text": "What is the capital of France?", # Original prompt "response": '{"id": "test-123", "choices": [{"message": {"content": "Paris is the capital of France."}}]}', }, @@ -127,6 +124,67 @@ def test_qdrant_semantic_cache_get_cache_hit(): # Verify search was called qdrant_cache.sync_client.post.assert_called() + assert qdrant_cache.sync_client.post.call_args.kwargs["json"]["filter"] == { + "must": [ + { + "key": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "match": {"value": "test_key"}, + } + ] + } + + +def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): + """ + Test QDRANT semantic cache rejects old or unscoped cache hits. + """ + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), + ): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = { + "result": [ + { + "payload": { + "text": "What is the capital of France?", + "response": '{"id": "test-123"}', + }, + "score": 0.9, + } + ] + } + qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", messages=[{"content": "What is the capital of France?"}] + ) + + assert result is None def test_qdrant_semantic_cache_get_cache_miss(): @@ -138,9 +196,7 @@ def test_qdrant_semantic_cache_get_cache_miss(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -230,6 +286,7 @@ async def test_qdrant_semantic_cache_async_get_cache_hit(): "result": [ { "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", "text": "What is the capital of Spain?", # Original prompt "response": '{"id": "test-456", "choices": [{"message": {"content": "Madrid is the capital of Spain."}}]}', }, @@ -262,6 +319,16 @@ async def test_qdrant_semantic_cache_async_get_cache_hit(): # Verify async search was called qdrant_cache.async_client.post.assert_called() + assert qdrant_cache.async_client.post.call_args.kwargs["json"][ + "filter" + ] == { + "must": [ + { + "key": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "match": {"value": "test_key"}, + } + ] + } @pytest.mark.asyncio @@ -336,9 +403,7 @@ def test_qdrant_semantic_cache_set_cache(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -384,6 +449,12 @@ def test_qdrant_semantic_cache_set_cache(): # Verify upsert was called qdrant_cache.sync_client.put.assert_called() + upsert_payload = qdrant_cache.sync_client.put.call_args.kwargs["json"][ + "points" + ][0]["payload"] + assert ( + upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" + ) @pytest.mark.asyncio @@ -450,6 +521,12 @@ async def test_qdrant_semantic_cache_async_set_cache(): # Verify async upsert was called qdrant_cache.async_client.put.assert_called() + upsert_payload = qdrant_cache.async_client.put.call_args.kwargs["json"][ + "points" + ][0]["payload"] + assert ( + upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" + ) def test_qdrant_semantic_cache_custom_vector_size(): @@ -462,9 +539,7 @@ def test_qdrant_semantic_cache_custom_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection does NOT exist (so it will be created) @@ -521,9 +596,7 @@ def test_qdrant_semantic_cache_default_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -559,9 +632,7 @@ def test_qdrant_semantic_cache_large_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection does NOT exist (so it will be created) diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index f9946e266f..9f2f390f4e 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -72,13 +72,22 @@ def test_redis_semantic_cache_get_cache(monkeypatch): "prompt": "What is the capital of France?", "response": '{"content": "Paris is the capital of France."}', "vector_distance": 0.1, # Distance of 0.1 means similarity of 0.9 + RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", } ] redis_semantic_cache.llmcache.check = MagicMock(return_value=mock_result) # Mock the embedding function - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + with ( + patch( + "litellm.embedding", + return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]}, + ), + patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ), ): # Test get_cache with a message result = redis_semantic_cache.get_cache( @@ -89,7 +98,131 @@ def test_redis_semantic_cache_get_cache(monkeypatch): assert result == {"content": "Paris is the capital of France."} # Verify llmcache.check was called - redis_semantic_cache.llmcache.check.assert_called_once() + redis_semantic_cache.llmcache.check.assert_called_once_with( + prompt="What is the capital of France?", + filter_expression="cache-key-filter", + ) + + +def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.check = MagicMock( + return_value=[ + { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + ] + ) + + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + result = redis_semantic_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + ) + + assert result is None + + +def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.store = MagicMock() + + redis_semantic_cache.set_cache( + key="test_key", + value={"content": "Paris"}, + messages=[{"content": "What is the capital of France?"}], + ttl=60, + ) + + redis_semantic_cache.llmcache.store.assert_called_once_with( + "What is the capital of France?", + "{'content': 'Paris'}", + filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, + ttl=60, + ) + + +def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert semantic_cache_mock.call_args_list[0].kwargs["name"] == "existing_index" + assert ( + semantic_cache_mock.call_args_list[1].kwargs["name"] + == "existing_index_isolated" + ) + assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ + RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + ] @pytest.mark.asyncio @@ -123,6 +256,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): "prompt": "What is the capital of France?", "response": '{"content": "Paris is the capital of France."}', "vector_distance": 0.1, # Distance of 0.1 means similarity of 0.9 + RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", } ] @@ -131,16 +265,69 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): return_value=[0.1, 0.2, 0.3] ) - # Test async_get_cache with a message - result = await redis_semantic_cache.async_get_cache( - key="test_key", - messages=[{"content": "What is the capital of France?"}], - metadata={}, - ) + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + # Test async_get_cache with a message + result = await redis_semantic_cache.async_get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata={}, + ) # Verify result is properly parsed assert result == {"content": "Paris is the capital of France."} # Verify methods were called redis_semantic_cache._get_async_embedding.assert_called_once() - redis_semantic_cache.llmcache.acheck.assert_called_once() + redis_semantic_cache.llmcache.acheck.assert_called_once_with( + prompt="What is the capital of France?", + vector=[0.1, 0.2, 0.3], + filter_expression="cache-key-filter", + ) + + +@pytest.mark.asyncio +async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( + monkeypatch, +): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.astore = AsyncMock() + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) + + await redis_semantic_cache.async_set_cache( + key="test_key", + value={"content": "Paris"}, + messages=[{"content": "What is the capital of France?"}], + ttl=60, + ) + + redis_semantic_cache.llmcache.astore.assert_called_once_with( + "What is the capital of France?", + "{'content': 'Paris'}", + vector=[0.1, 0.2, 0.3], + filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, + ttl=60, + ) From 9aa3dfc816f5472dc9ef28854c6198109b1ada69 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 01:10:44 -0700 Subject: [PATCH 02/12] chore(proxy): stabilize lazy openapi snapshot --- litellm/proxy/_lazy_openapi_snapshot.json | 28 ++++++------- litellm/proxy/_lazy_openapi_snapshot.py | 40 ++++++++++++++++++- .../proxy/test_lazy_openapi_snapshot.py | 35 ++++++++++++++++ 3 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 tests/test_litellm/proxy/test_lazy_openapi_snapshot.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8331f748c6..46a514c087 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3572,7 +3572,7 @@ "/anthropic/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__delete", "parameters": [ { "in": "path", @@ -3616,7 +3616,7 @@ }, "get": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__get", "parameters": [ { "in": "path", @@ -3660,7 +3660,7 @@ }, "patch": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__patch", "parameters": [ { "in": "path", @@ -3704,7 +3704,7 @@ }, "post": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__post", "parameters": [ { "in": "path", @@ -13260,7 +13260,7 @@ "/langfuse/{endpoint}": { "delete": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__delete", "parameters": [ { "in": "path", @@ -13299,7 +13299,7 @@ }, "get": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__get", "parameters": [ { "in": "path", @@ -13338,7 +13338,7 @@ }, "patch": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__patch", "parameters": [ { "in": "path", @@ -13377,7 +13377,7 @@ }, "post": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__post", "parameters": [ { "in": "path", @@ -26883,7 +26883,7 @@ "/toolset/{toolset_name}/mcp": { "delete": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete", "parameters": [ { "in": "path", @@ -26922,7 +26922,7 @@ }, "get": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get", "parameters": [ { "in": "path", @@ -26961,7 +26961,7 @@ }, "head": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head", "parameters": [ { "in": "path", @@ -27000,7 +27000,7 @@ }, "options": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options", "parameters": [ { "in": "path", @@ -27039,7 +27039,7 @@ }, "patch": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch", "parameters": [ { "in": "path", @@ -27078,7 +27078,7 @@ }, "post": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post", "parameters": [ { "in": "path", diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 315f6a9742..309a0276aa 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Dict, Optional SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json" +HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"} def load_snapshot() -> Optional[Dict[str, Dict]]: @@ -25,6 +26,39 @@ def load_snapshot() -> Optional[Dict[str, Dict]]: return None +def _normalize_operation_ids(paths: Dict[str, Dict]) -> None: + """Make FastAPI-generated operation IDs stable for multi-method routes. + + FastAPI derives the default operation ID suffix from the first item in the + route's methods set. For routes registered with several HTTP methods, that + set iteration order can vary between processes, which makes the snapshot + drift even when no routes changed. + """ + for path_ops in paths.values(): + if not isinstance(path_ops, dict): + continue + + methods = {method for method in path_ops if method in HTTP_METHODS} + if not methods: + continue + + for method, operation in path_ops.items(): + if method not in HTTP_METHODS or not isinstance(operation, dict): + continue + + operation_id = operation.get("operationId") + if not isinstance(operation_id, str): + continue + + for suffix in methods: + suffix_token = f"_{suffix}" + if operation_id.endswith(suffix_token): + operation["operationId"] = ( + operation_id[: -len(suffix_token)] + f"_{method}" + ) + break + + def generate_snapshot() -> Dict[str, Dict]: import importlib @@ -52,13 +86,15 @@ def generate_snapshot() -> Dict[str, Dict]: if not feat_routes: continue full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + paths = full.get("paths", {}) + _normalize_operation_ids(paths) # Group all of a feature's routes under one tag. - for path_ops in full.get("paths", {}).values(): + for path_ops in paths.values(): for op in path_ops.values(): if isinstance(op, dict): op["tags"] = [feat.name] fragments[feat.name] = { - "paths": full.get("paths", {}), + "paths": paths, "components": {"schemas": full.get("components", {}).get("schemas", {})}, } return fragments diff --git a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py new file mode 100644 index 0000000000..8bc39c93ee --- /dev/null +++ b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py @@ -0,0 +1,35 @@ +from litellm.proxy._lazy_openapi_snapshot import _normalize_operation_ids + + +def test_normalize_operation_ids_uses_each_http_method(): + paths = { + "/proxy/{endpoint}": { + "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, + "get": {"operationId": "proxy_route_proxy__endpoint__put"}, + "post": {"operationId": "proxy_route_proxy__endpoint__put"}, + "put": {"operationId": "proxy_route_proxy__endpoint__put"}, + } + } + + _normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" + assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" + assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" + assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" + + +def test_normalize_operation_ids_preserves_custom_ids(): + paths = { + "/proxy/{endpoint}": { + "get": {"operationId": "custom_operation"}, + "post": {"operationId": "custom_operation"}, + } + } + + _normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["get"]["operationId"] == "custom_operation" + assert operations["post"]["operationId"] == "custom_operation" From ae9b63c468bcca396ec0b1e82a93df0fae86e065 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:15:52 -0700 Subject: [PATCH 03/12] chore(caching): index qdrant semantic cache scope --- litellm/caching/qdrant_semantic_cache.py | 26 ++++++++++++++ .../caching/test_qdrant_semantic_cache.py | 36 ++++++++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index b856423cbb..e6e8e25522 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -120,6 +120,7 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"Collection already exists.\nCollection details:{self.collection_info}" ) + self._ensure_cache_key_payload_index() else: if quantization_config is None or quantization_config == "binary": quantization_params = { @@ -161,6 +162,7 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"New collection created.\nCollection details:{self.collection_info}" ) + self._ensure_cache_key_payload_index() else: raise Exception("Error while creating new collection") @@ -185,7 +187,31 @@ class QdrantSemanticCache(BaseCache): ] } + def _ensure_cache_key_payload_index(self) -> None: + try: + response = self.sync_client.put( + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/index", + headers=self.headers, + json={ + "field_name": self.CACHE_KEY_FIELD_NAME, + "field_schema": "keyword", + }, + ) + if response.status_code not in (200, 201): + print_verbose( + "Qdrant semantic-cache could not create cache-key payload index: " + f"{response.text}" + ) + except Exception as exc: + print_verbose( + "Qdrant semantic-cache could not create cache-key payload index: " + f"{str(exc)}" + ) + def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: + # Legacy Qdrant semantic-cache points stored only prompt text and + # response. They cannot be reassigned to the generated LiteLLM cache key + # without risking cross-scope hits, so they must be treated as misses. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) return cached_key is not None and str(cached_key) == str(key) diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index f357d97341..733cf47c89 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -29,6 +29,9 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): mock_sync_client_instance = MagicMock() mock_sync_client_instance.get.return_value = mock_response + mock_index_response = MagicMock() + mock_index_response.status_code = 200 + mock_sync_client_instance.put.return_value = mock_index_response mock_sync_client.return_value = mock_sync_client_instance from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache @@ -46,6 +49,17 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): assert qdrant_cache.qdrant_api_base == "http://test.qdrant.local" assert qdrant_cache.qdrant_api_key == "test_key" assert qdrant_cache.similarity_threshold == 0.8 + mock_sync_client_instance.put.assert_called_once_with( + url="http://test.qdrant.local/collections/test_collection/index", + headers={ + "Content-Type": "application/json", + "api-key": "test_key", + }, + json={ + "field_name": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "field_schema": "keyword", + }, + ) # Test initialization with missing similarity_threshold with pytest.raises(Exception, match="similarity_threshold must be provided"): @@ -137,6 +151,9 @@ def test_qdrant_semantic_cache_get_cache_hit(): def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): """ Test QDRANT semantic cache rejects old or unscoped cache hits. + + Legacy points have only text and response payloads, so they cannot be + safely migrated to a generated LiteLLM cache key. """ with ( patch( @@ -580,9 +597,13 @@ def test_qdrant_semantic_cache_custom_vector_size(): assert qdrant_cache.vector_size == 768 # Verify the PUT call to create the collection used vector_size=768 - put_call = mock_sync_client_instance.put.call_args - assert put_call is not None - create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + put_call = next( + call + for call in mock_sync_client_instance.put.call_args_list + if call.kwargs["url"] + == "http://test.qdrant.local/collections/test_collection_768" + ) + create_payload = put_call.kwargs["json"] assert create_payload["vectors"]["size"] == 768 assert create_payload["vectors"]["distance"] == "Cosine" @@ -670,6 +691,11 @@ def test_qdrant_semantic_cache_large_vector_size(): assert qdrant_cache.vector_size == 4096 # Verify the collection was created with 4096 - put_call = mock_sync_client_instance.put.call_args - create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + put_call = next( + call + for call in mock_sync_client_instance.put.call_args_list + if call.kwargs["url"] + == "http://test.qdrant.local/collections/test_collection_4096" + ) + create_payload = put_call.kwargs["json"] assert create_payload["vectors"]["size"] == 4096 From 1c19bdda79447249e81f9530e072941c577e844f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:26:03 -0700 Subject: [PATCH 04/12] test(caching): cover semantic cache isolation guards --- litellm/proxy/_lazy_openapi_snapshot.py | 4 +- .../caching/test_qdrant_semantic_cache.py | 33 +++++++ .../caching/test_redis_semantic_cache.py | 94 +++++++++++++++++++ .../proxy/test_lazy_openapi_snapshot.py | 35 ------- .../test_lazy_openapi_snapshot.py | 76 +++++++++++++++ 5 files changed, 205 insertions(+), 37 deletions(-) delete mode 100644 tests/test_litellm/proxy/test_lazy_openapi_snapshot.py create mode 100644 tests/test_litellm/test_lazy_openapi_snapshot.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 309a0276aa..eb32443baa 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -59,7 +59,7 @@ def _normalize_operation_ids(paths: Dict[str, Dict]) -> None: break -def generate_snapshot() -> Dict[str, Dict]: +def generate_snapshot() -> Dict[str, Dict]: # pragma: no cover import importlib from fastapi.openapi.utils import get_openapi @@ -100,7 +100,7 @@ def generate_snapshot() -> Dict[str, Dict]: return fragments -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover fragments = generate_snapshot() SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n") sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n") diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 733cf47c89..aa46d18023 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -204,6 +204,39 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert result is None +def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = {"Content-Type": "application/json"} + qdrant_cache.sync_client = MagicMock() + response = MagicMock() + response.status_code = 400 + response.text = "bad index" + qdrant_cache.sync_client.put.return_value = response + + qdrant_cache._ensure_cache_key_payload_index() + + qdrant_cache.sync_client.put.assert_called_once() + + +def test_qdrant_semantic_cache_payload_index_exception_is_non_blocking(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = {"Content-Type": "application/json"} + qdrant_cache.sync_client = MagicMock() + qdrant_cache.sync_client.put.side_effect = Exception("boom") + + qdrant_cache._ensure_cache_key_payload_index() + + qdrant_cache.sync_client.put.assert_called_once() + + def test_qdrant_semantic_cache_get_cache_miss(): """ Test QDRANT semantic cache get method when there's a cache miss. diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 9f2f390f4e..b702de8d5f 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -225,6 +225,52 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] +def test_redis_semantic_cache_reraises_unexpected_index_error(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache.distance_threshold = 0.2 + semantic_cache_mock = MagicMock(side_effect=ValueError("connection failed")) + + with pytest.raises(ValueError, match="connection failed"): + redis_semantic_cache._init_semantic_cache( + semantic_cache_cls=semantic_cache_mock, + index_name="existing_index", + redis_url="redis://localhost:6379", + cache_vectorizer=MagicMock(), + ) + + +def test_redis_semantic_cache_matches_bytes_cache_key(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + + assert redis_semantic_cache._cache_hit_matches_key( + cache_hit={RedisSemanticCache.CACHE_KEY_FIELD_NAME: b"test_key"}, + key="test_key", + ) + + +def test_redis_semantic_cache_builds_filter_expression(monkeypatch): + class FakeTag: + def __init__(self, field_name): + self.field_name = field_name + + def __eq__(self, value): + return (self.field_name, value) + + with patch.dict("sys.modules", {"redisvl.query.filter": MagicMock(Tag=FakeTag)}): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + + assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( + RedisSemanticCache.CACHE_KEY_FIELD_NAME, + "test_key", + ) + + @pytest.mark.asyncio async def test_redis_semantic_cache_async_get_cache(monkeypatch): # Mock the redisvl import @@ -289,6 +335,54 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): ) +@pytest.mark.asyncio +async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.acheck = AsyncMock( + return_value=[ + { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + ] + ) + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) + + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + result = await redis_semantic_cache.async_get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata={}, + ) + + assert result is None + + @pytest.mark.asyncio async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( monkeypatch, diff --git a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py deleted file mode 100644 index 8bc39c93ee..0000000000 --- a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py +++ /dev/null @@ -1,35 +0,0 @@ -from litellm.proxy._lazy_openapi_snapshot import _normalize_operation_ids - - -def test_normalize_operation_ids_uses_each_http_method(): - paths = { - "/proxy/{endpoint}": { - "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, - "get": {"operationId": "proxy_route_proxy__endpoint__put"}, - "post": {"operationId": "proxy_route_proxy__endpoint__put"}, - "put": {"operationId": "proxy_route_proxy__endpoint__put"}, - } - } - - _normalize_operation_ids(paths) - - operations = paths["/proxy/{endpoint}"] - assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" - assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" - assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" - assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" - - -def test_normalize_operation_ids_preserves_custom_ids(): - paths = { - "/proxy/{endpoint}": { - "get": {"operationId": "custom_operation"}, - "post": {"operationId": "custom_operation"}, - } - } - - _normalize_operation_ids(paths) - - operations = paths["/proxy/{endpoint}"] - assert operations["get"]["operationId"] == "custom_operation" - assert operations["post"]["operationId"] == "custom_operation" diff --git a/tests/test_litellm/test_lazy_openapi_snapshot.py b/tests/test_litellm/test_lazy_openapi_snapshot.py new file mode 100644 index 0000000000..c33bab2c75 --- /dev/null +++ b/tests/test_litellm/test_lazy_openapi_snapshot.py @@ -0,0 +1,76 @@ +from litellm.proxy import _lazy_openapi_snapshot as snapshot_module + + +def test_load_snapshot_returns_none_when_missing(monkeypatch, tmp_path): + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", tmp_path / "missing.json") + + assert snapshot_module.load_snapshot() is None + + +def test_load_snapshot_reads_json(monkeypatch, tmp_path): + snapshot_file = tmp_path / "snapshot.json" + snapshot_file.write_text('{"mcp": {"paths": {}}}') + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", snapshot_file) + + assert snapshot_module.load_snapshot() == {"mcp": {"paths": {}}} + + +def test_load_snapshot_returns_none_for_invalid_json(monkeypatch, tmp_path): + snapshot_file = tmp_path / "snapshot.json" + snapshot_file.write_text("{") + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", snapshot_file) + + assert snapshot_module.load_snapshot() is None + + +def test_normalize_operation_ids_uses_each_http_method(): + paths = { + "/proxy/{endpoint}": { + "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, + "get": {"operationId": "proxy_route_proxy__endpoint__put"}, + "post": {"operationId": "proxy_route_proxy__endpoint__put"}, + "put": {"operationId": "proxy_route_proxy__endpoint__put"}, + } + } + + snapshot_module._normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" + assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" + assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" + assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" + + +def test_normalize_operation_ids_preserves_custom_ids(): + paths = { + "/proxy/{endpoint}": { + "get": {"operationId": "custom_operation"}, + "post": {"operationId": "custom_operation"}, + } + } + + snapshot_module._normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["get"]["operationId"] == "custom_operation" + assert operations["post"]["operationId"] == "custom_operation" + + +def test_normalize_operation_ids_skips_invalid_entries(): + paths = { + "/not-a-dict": "skip", + "/no-http-methods": {"parameters": []}, + "/invalid-operation": { + "get": ["skip"], + "post": {"operationId": 123}, + "parameters": [], + }, + } + + snapshot_module._normalize_operation_ids(paths) + + assert paths["/not-a-dict"] == "skip" + assert paths["/no-http-methods"] == {"parameters": []} + assert paths["/invalid-operation"]["get"] == ["skip"] + assert paths["/invalid-operation"]["post"] == {"operationId": 123} From 74e93444cf0b08198bcd5ee1dc1a822009211c04 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:57:41 -0700 Subject: [PATCH 05/12] chore(caching): align qdrant scoped miss metadata --- litellm/caching/qdrant_semantic_cache.py | 1 + tests/test_litellm/caching/test_qdrant_semantic_cache.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index e6e8e25522..e7436e2ca6 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -349,6 +349,7 @@ class QdrantSemanticCache(BaseCache): payload = results[0]["payload"] if not self._payload_matches_cache_key(payload=payload, key=key): print_verbose("Qdrant semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None cached_prompt = payload["text"] diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index aa46d18023..71dbd24063 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -197,11 +197,15 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): with patch( "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} ): + metadata = {} result = qdrant_cache.get_cache( - key="test_key", messages=[{"content": "What is the capital of France?"}] + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) assert result is None + assert metadata["semantic-similarity"] == 0.0 def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): From d8c11f962296feb8c7489bf6115062984e9716f7 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 12:10:00 -0700 Subject: [PATCH 06/12] chore(caching): align redis semantic miss metadata --- litellm/caching/redis_semantic_cache.py | 7 +++++++ tests/test_litellm/caching/test_redis_semantic_cache.py | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 06e9b4d0fa..85d3a4f1b4 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -290,6 +290,7 @@ class RedisSemanticCache(BaseCache): messages = kwargs.get("messages", []) if not messages: print_verbose("No messages provided for semantic cache lookup") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None prompt = get_str_from_messages(messages) @@ -302,12 +303,14 @@ class RedisSemanticCache(BaseCache): # Return None if no similar prompts found if not results: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None # Process the best matching result cache_hit = results[0] if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): print_verbose("Redis semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None vector_distance = float(cache_hit["vector_distance"]) @@ -319,6 +322,9 @@ class RedisSemanticCache(BaseCache): cached_prompt = cache_hit["prompt"] cached_response = cache_hit["response"] + # update kwargs["metadata"] with similarity, don't rewrite the original metadata + kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity + print_verbose( f"Cache hit: similarity threshold: {self.similarity_threshold}, " f"actual similarity: {similarity}, " @@ -329,6 +335,7 @@ class RedisSemanticCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: print_verbose(f"Error retrieving from Redis semantic cache: {str(e)}") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def _get_async_embedding(self, prompt: str, **kwargs) -> List[float]: """ diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index b702de8d5f..f5c84c9d3e 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -90,12 +90,16 @@ def test_redis_semantic_cache_get_cache(monkeypatch): ), ): # Test get_cache with a message + metadata = {} result = redis_semantic_cache.get_cache( - key="test_key", messages=[{"content": "What is the capital of France?"}] + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) # Verify result is properly parsed assert result == {"content": "Paris is the capital of France."} + assert metadata["semantic-similarity"] == pytest.approx(0.9) # Verify llmcache.check was called redis_semantic_cache.llmcache.check.assert_called_once_with( @@ -139,12 +143,15 @@ def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): "_get_cache_key_filter_expression", return_value="cache-key-filter", ): + metadata = {} result = redis_semantic_cache.get_cache( key="test_key", messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) assert result is None + assert metadata["semantic-similarity"] == 0.0 def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): From 8cb52ce0bbcae4d1e421796224543a4c118fa41f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 22:15:03 -0700 Subject: [PATCH 07/12] fix(caching): handle stale isolated Redis semantic index --- litellm/caching/redis_semantic_cache.py | 38 +++++++--- .../caching/test_redis_semantic_cache.py | 73 +++++++++++++++++++ 2 files changed, 102 insertions(+), 9 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 85d3a4f1b4..27538530bc 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -128,6 +128,9 @@ class RedisSemanticCache(BaseCache): redis_url: str, cache_vectorizer: Any, ) -> Any: + def _is_schema_mismatch(exc: ValueError) -> bool: + return "schema does not match" in str(exc) + try: return semantic_cache_cls( name=index_name, @@ -138,7 +141,7 @@ class RedisSemanticCache(BaseCache): overwrite=False, ) except ValueError as exc: - if "schema does not match" not in str(exc): + if not _is_schema_mismatch(exc): raise isolated_index_name = f"{index_name}_isolated" @@ -146,14 +149,31 @@ class RedisSemanticCache(BaseCache): "Redis semantic-cache existing index schema is not isolated; " f"using isolated index - {isolated_index_name}" ) - return semantic_cache_cls( - name=isolated_index_name, - redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], - overwrite=False, - ) + try: + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + except ValueError as isolated_exc: + if not _is_schema_mismatch(isolated_exc): + raise + + print_verbose( + "Redis semantic-cache isolated index schema is stale; " + f"recreating isolated index - {isolated_index_name}" + ) + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=True, + ) def _get_cache_filters(self, key: str) -> Dict[str, str]: return {self.CACHE_KEY_FIELD_NAME: str(key)} diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index f5c84c9d3e..02b06bed62 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -232,6 +232,79 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] +def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert ( + semantic_cache_mock.call_args_list[2].kwargs["name"] + == "existing_index_isolated" + ) + assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True + assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ + RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + ] + + +def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypatch): + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + ValueError("connection failed"), + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + with pytest.raises(ValueError, match="connection failed"): + RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + def test_redis_semantic_cache_reraises_unexpected_index_error(): from litellm.caching.redis_semantic_cache import RedisSemanticCache From a05d3b585170a5db361ece0c1f227c134686ac3d Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 11:18:41 -0700 Subject: [PATCH 08/12] Fix qdrant semantic cache miss metadata --- litellm/caching/qdrant_semantic_cache.py | 6 ++ .../caching/test_qdrant_semantic_cache.py | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index e7436e2ca6..47359a6023 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -340,9 +340,11 @@ class QdrantSemanticCache(BaseCache): results = search_response.json()["result"] if results is None: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None if isinstance(results, list): if len(results) == 0: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None similarity = results[0]["score"] @@ -358,6 +360,10 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"semantic cache: similarity threshold: {self.similarity_threshold}, similarity: {similarity}, prompt: {prompt}, closest_cached_prompt: {cached_prompt}" ) + + # update kwargs["metadata"] with similarity, don't rewrite the original metadata + kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity + if similarity >= self.similarity_threshold: # cache hit ! cached_value = payload["response"] diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 71dbd24063..949e6ccc29 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -241,6 +241,76 @@ def test_qdrant_semantic_cache_payload_index_exception_is_non_blocking(): qdrant_cache.sync_client.put.assert_called_once() +def _mock_qdrant_get_cache_result(qdrant_result): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.embedding_model = "text-embedding-ada-002" + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = { + "Content-Type": "application/json", + "api-key": "test_key", + } + qdrant_cache.similarity_threshold = 0.8 + qdrant_cache.sync_client = MagicMock() + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = {"result": qdrant_result} + qdrant_cache.sync_client.post.return_value = mock_search_response + + return qdrant_cache, QdrantSemanticCache + + +@pytest.mark.parametrize("qdrant_result", [None, []]) +def test_qdrant_semantic_cache_get_cache_sets_metadata_on_empty_miss(qdrant_result): + qdrant_cache, _ = _mock_qdrant_get_cache_result(qdrant_result) + metadata = {} + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of Spain?"}], + metadata=metadata, + ) + + assert result is None + assert metadata["semantic-similarity"] == 0.0 + + +def test_qdrant_semantic_cache_get_cache_sets_metadata_on_below_threshold_miss(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache, _ = _mock_qdrant_get_cache_result( + [ + { + "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", + "text": "What is the capital of Spain?", + "response": '{"id": "test-456"}', + }, + "score": 0.7, + } + ] + ) + metadata = {} + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of Spain?"}], + metadata=metadata, + ) + + assert result is None + assert metadata["semantic-similarity"] == 0.7 + + def test_qdrant_semantic_cache_get_cache_miss(): """ Test QDRANT semantic cache get method when there's a cache miss. From 9f1feaadeb97161cf2d1d9c750e0f523c3b6a202 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 14:23:31 -0700 Subject: [PATCH 09/12] Clean up Redis semantic cache isolation fallback --- litellm/caching/redis_semantic_cache.py | 30 ++++++++----------- .../caching/test_redis_semantic_cache.py | 2 +- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 27538530bc..4d685c6a87 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -129,7 +129,11 @@ class RedisSemanticCache(BaseCache): cache_vectorizer: Any, ) -> Any: def _is_schema_mismatch(exc: ValueError) -> bool: - return "schema does not match" in str(exc) + error_message = str(exc).lower() + return any( + phrase in error_message + for phrase in ("schema does not match", "index schema") + ) try: return semantic_cache_cls( @@ -284,9 +288,7 @@ class RedisSemanticCache(BaseCache): ttl = self._get_ttl(**kwargs) if ttl is not None: store_kwargs["ttl"] = int(ttl) - self.llmcache.store(prompt, value_str, **store_kwargs) - else: - self.llmcache.store(prompt, value_str, **store_kwargs) + self.llmcache.store(prompt, value_str, **store_kwargs) except Exception as e: print_verbose( f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}" @@ -439,17 +441,11 @@ class RedisSemanticCache(BaseCache): ttl = self._get_ttl(**kwargs) if ttl is not None: store_kwargs["ttl"] = ttl - await self.llmcache.astore( - prompt, - value_str, - **store_kwargs, - ) - else: - await self.llmcache.astore( - prompt, - value_str, - **store_kwargs, - ) + await self.llmcache.astore( + prompt, + value_str, + **store_kwargs, + ) except Exception as e: print_verbose(f"Error in async_set_cache: {str(e)}") @@ -489,9 +485,7 @@ class RedisSemanticCache(BaseCache): # handle results / cache hit if not results: - kwargs.setdefault("metadata", {})[ - "semantic-similarity" - ] = 0.0 # TODO why here but not above?? + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None cache_hit = results[0] diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 02b06bed62..495c72a380 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -195,7 +195,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( side_effect=[ - ValueError("Existing index schema does not match"), + ValueError("stored index schema differs from requested fields"), fallback_cache_mock, ] ) From af7794272b92354f267e6c05c0f0c64660356037 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 14:42:53 -0700 Subject: [PATCH 10/12] Add semantic cache legacy migration flag --- litellm/caching/qdrant_semantic_cache.py | 42 ++++++++-- litellm/caching/redis_semantic_cache.py | 82 +++++++++++++++---- .../caching/test_qdrant_semantic_cache.py | 70 ++++++++++++++++ .../caching/test_redis_semantic_cache.py | 67 ++++++++++++++- 4 files changed, 235 insertions(+), 26 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 47359a6023..a9f01a4c90 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -11,7 +11,8 @@ Has 4 methods: import ast import asyncio import json -from typing import Any, cast +import os +from typing import Any, Optional, cast import litellm from litellm._logging import print_verbose @@ -26,6 +27,9 @@ from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): CACHE_KEY_FIELD_NAME = "litellm_cache_key" + ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR = ( + "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" + ) def __init__( # noqa: PLR0915 self, @@ -37,9 +41,8 @@ class QdrantSemanticCache(BaseCache): embedding_model="text-embedding-ada-002", host_type=None, vector_size=None, + allow_legacy_unscoped_cache_hits: Optional[bool] = None, ): - import os - from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -59,6 +62,16 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model + self.allow_legacy_unscoped_cache_hits = ( + self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) + ) + if self.allow_legacy_unscoped_cache_hits: + print_verbose( + "Qdrant semantic-cache legacy unscoped hits are enabled via " + f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; searches may return " + "pre-isolation cache entries without cache-key payloads. Disable " + "this after warming the key-scoped semantic cache." + ) self.vector_size = ( vector_size if vector_size is not None else QDRANT_VECTOR_SIZE ) @@ -166,6 +179,14 @@ class QdrantSemanticCache(BaseCache): else: raise Exception("Error while creating new collection") + @classmethod + def _get_allow_legacy_unscoped_cache_hits( + cls, allow_legacy_unscoped_cache_hits: Optional[bool] + ) -> bool: + if allow_legacy_unscoped_cache_hits is not None: + return allow_legacy_unscoped_cache_hits + return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" + def _get_cache_logic(self, cached_response: Any): if cached_response is None: return cached_response @@ -187,6 +208,11 @@ class QdrantSemanticCache(BaseCache): ] } + def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None: + if getattr(self, "allow_legacy_unscoped_cache_hits", False): + return + data["filter"] = self._get_qdrant_cache_key_filter(key) + def _ensure_cache_key_payload_index(self) -> None: try: response = self.sync_client.put( @@ -211,8 +237,12 @@ class QdrantSemanticCache(BaseCache): def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: # Legacy Qdrant semantic-cache points stored only prompt text and # response. They cannot be reassigned to the generated LiteLLM cache key - # without risking cross-scope hits, so they must be treated as misses. + # without risking cross-scope hits, so secure mode treats them as misses. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) + if cached_key is None and getattr( + self, "allow_legacy_unscoped_cache_hits", False + ): + return True return cached_key is not None and str(cached_key) == str(key) async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: @@ -329,8 +359,8 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, - "filter": self._get_qdrant_cache_key_filter(key), } + self._add_cache_key_filter_to_search_data(data=data, key=key) search_response = self.sync_client.post( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", @@ -436,8 +466,8 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, - "filter": self._get_qdrant_cache_key_filter(key), } + self._add_cache_key_filter_to_search_data(data=data, key=key) search_response = await self.async_client.post( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 4d685c6a87..35fed977cc 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -36,10 +36,9 @@ class RedisSemanticCache(BaseCache): DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" - CACHE_KEY_FILTERABLE_FIELD: Dict[str, str] = { - "name": CACHE_KEY_FIELD_NAME, - "type": "tag", - } + ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR: str = ( + "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" + ) def __init__( self, @@ -50,6 +49,7 @@ class RedisSemanticCache(BaseCache): similarity_threshold: Optional[float] = None, embedding_model: str = "text-embedding-ada-002", index_name: Optional[str] = None, + allow_legacy_unscoped_cache_hits: Optional[bool] = None, **kwargs, ): """ @@ -91,6 +91,10 @@ class RedisSemanticCache(BaseCache): # While similarity: 1 = most similar, 0 = least similar self.distance_threshold = 1 - similarity_threshold self.embedding_model = embedding_model + self.allow_legacy_unscoped_cache_hits = ( + self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) + ) + self._using_legacy_unscoped_index = False # Set up Redis connection if redis_url is None: @@ -121,6 +125,21 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) + @classmethod + def _get_allow_legacy_unscoped_cache_hits( + cls, allow_legacy_unscoped_cache_hits: Optional[bool] + ) -> bool: + if allow_legacy_unscoped_cache_hits is not None: + return allow_legacy_unscoped_cache_hits + return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" + + @classmethod + def _cache_key_filterable_field(cls) -> Dict[str, str]: + return { + "name": cls.CACHE_KEY_FIELD_NAME, + "type": "tag", + } + def _init_semantic_cache( self, semantic_cache_cls: Any, @@ -141,13 +160,29 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=False, ) except ValueError as exc: if not _is_schema_mismatch(exc): raise + if self.allow_legacy_unscoped_cache_hits: + self._using_legacy_unscoped_index = True + print_verbose( + "Redis semantic-cache legacy unscoped hits are enabled via " + f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; reusing existing " + "index without cache-key isolation. Disable this after warming " + "the isolated semantic cache." + ) + return semantic_cache_cls( + name=index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + overwrite=False, + ) + isolated_index_name = f"{index_name}_isolated" print_verbose( "Redis semantic-cache existing index schema is not isolated; " @@ -159,7 +194,7 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=False, ) except ValueError as isolated_exc: @@ -175,7 +210,7 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=True, ) @@ -191,6 +226,8 @@ class RedisSemanticCache(BaseCache): cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) if isinstance(cached_key, bytes): cached_key = cached_key.decode("utf-8") + if cached_key is None and getattr(self, "_using_legacy_unscoped_index", False): + return True return cached_key is not None and str(cached_key) == str(key) def _get_ttl(self, **kwargs) -> Optional[int]: @@ -282,7 +319,9 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) - store_kwargs: Dict[str, Any] = {"filters": self._get_cache_filters(key)} + store_kwargs: Dict[str, Any] = {} + if not getattr(self, "_using_legacy_unscoped_index", False): + store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -318,10 +357,12 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - results = self.llmcache.check( - prompt=prompt, - filter_expression=self._get_cache_key_filter_expression(key), - ) + check_kwargs: Dict[str, Any] = {"prompt": prompt} + if not getattr(self, "_using_legacy_unscoped_index", False): + check_kwargs["filter_expression"] = ( + self._get_cache_key_filter_expression(key) + ) + results = self.llmcache.check(**check_kwargs) # Return None if no similar prompts found if not results: @@ -434,8 +475,9 @@ class RedisSemanticCache(BaseCache): store_kwargs: Dict[str, Any] = { "vector": prompt_embedding, - "filters": self._get_cache_filters(key), } + if not getattr(self, "_using_legacy_unscoped_index", False): + store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -477,11 +519,15 @@ class RedisSemanticCache(BaseCache): # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - results = await self.llmcache.acheck( - prompt=prompt, - vector=prompt_embedding, - filter_expression=self._get_cache_key_filter_expression(key), - ) + check_kwargs: Dict[str, Any] = { + "prompt": prompt, + "vector": prompt_embedding, + } + if not getattr(self, "_using_legacy_unscoped_index", False): + check_kwargs["filter_expression"] = ( + self._get_cache_key_filter_expression(key) + ) + results = await self.llmcache.acheck(**check_kwargs) # handle results / cache hit if not results: diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 949e6ccc29..9b987a6d4f 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -208,6 +208,76 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert metadata["semantic-similarity"] == 0.0 +def test_qdrant_semantic_cache_allows_legacy_unscoped_hit_with_flag(monkeypatch): + monkeypatch.setenv("LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS", "true") + + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), + ): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = { + "result": [ + { + "payload": { + "text": "What is the capital of France?", + "response": '{"id": "test-123"}', + }, + "score": 0.9, + } + ] + } + qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + metadata = {} + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, + ) + + assert result == {"id": "test-123"} + assert metadata["semantic-similarity"] == 0.9 + assert "filter" not in qdrant_cache.sync_client.post.call_args.kwargs["json"] + + +def test_qdrant_semantic_cache_legacy_mode_rejects_wrong_key_hit(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.allow_legacy_unscoped_cache_hits = True + + assert qdrant_cache._payload_matches_cache_key(payload={}, key="test_key") + assert not qdrant_cache._payload_matches_cache_key( + payload={QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "other_key"}, + key="test_key", + ) + + def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 495c72a380..bebe1f757b 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -228,10 +228,50 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): == "existing_index_isolated" ) assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ - RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + RedisSemanticCache._cache_key_filterable_field() ] +def test_redis_semantic_cache_can_reuse_legacy_unscoped_index(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + monkeypatch.setenv( + RedisSemanticCache.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "true" + ) + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert redis_semantic_cache._using_legacy_unscoped_index is True + assert semantic_cache_mock.call_count == 2 + assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index" + assert "filterable_fields" not in semantic_cache_mock.call_args_list[1].kwargs + + def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( @@ -270,7 +310,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ - RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + RedisSemanticCache._cache_key_filterable_field() ] @@ -332,6 +372,29 @@ def test_redis_semantic_cache_matches_bytes_cache_key(): ) +def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache._using_legacy_unscoped_index = False + + cache_hit = { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + assert not redis_semantic_cache._cache_hit_matches_key( + cache_hit=cache_hit, + key="test_key", + ) + + redis_semantic_cache._using_legacy_unscoped_index = True + assert redis_semantic_cache._cache_hit_matches_key( + cache_hit=cache_hit, + key="test_key", + ) + + def test_redis_semantic_cache_builds_filter_expression(monkeypatch): class FakeTag: def __init__(self, field_name): From 7d7244986ed242e6807142ee2559ddf64d7b5c6f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 22:10:59 +0000 Subject: [PATCH 11/12] chore(caching): annotate qdrant quantization_params dict type Mypy infers the dict's value type from the first branch (Dict[str, bool]) which clashes with the scalar branch's mixed-type inner dict. Explicit Dict[str, Any] annotation lifts the inference. --- litellm/caching/qdrant_semantic_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index a9f01a4c90..9206f326ec 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -12,7 +12,7 @@ import ast import asyncio import json import os -from typing import Any, Optional, cast +from typing import Any, Dict, Optional, cast import litellm from litellm._logging import print_verbose @@ -135,6 +135,7 @@ class QdrantSemanticCache(BaseCache): ) self._ensure_cache_key_payload_index() else: + quantization_params: Dict[str, Any] if quantization_config is None or quantization_config == "binary": quantization_params = { "binary": { From a2473ef0c23275bbfcee6a1a35cda684cf522492 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 22:16:30 +0000 Subject: [PATCH 12/12] chore(caching): remove allow_legacy_unscoped_cache_hits opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag was an opt-in escape hatch for the cross-tenant leak the rest of the patch closes — flipping it on (env var or constructor param) re-enables exactly the VERIA-54 primitive on either backend. There is no operational need that the secure path doesn't already meet: - Qdrant: legacy points without ``litellm_cache_key`` payload are excluded by the must-clause filter and treated as misses; new sets populate the cache key, so cold-start lasts only as long as the natural cache rebuild. - Redis: existing unscoped index can't carry the new schema; the init path falls back to ``{name}_isolated`` (and recreates it on stale schema), leaving the legacy index untouched. Drop the constructor param, env-var fallback, ``_using_legacy_unscoped_index`` flag, the legacy-reuse branch in ``_init_semantic_cache``, and the matching guards in set/get paths. Update tests to drop the legacy-mode cases and assert the secure-only behaviour. --- litellm/caching/qdrant_semantic_cache.py | 37 ++-------- litellm/caching/redis_semantic_cache.py | 59 +++------------- .../caching/test_qdrant_semantic_cache.py | 70 ------------------- .../caching/test_redis_semantic_cache.py | 51 +------------- 4 files changed, 19 insertions(+), 198 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 9206f326ec..cb521efca0 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -12,7 +12,7 @@ import ast import asyncio import json import os -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, cast import litellm from litellm._logging import print_verbose @@ -27,9 +27,6 @@ from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): CACHE_KEY_FIELD_NAME = "litellm_cache_key" - ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR = ( - "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" - ) def __init__( # noqa: PLR0915 self, @@ -41,7 +38,6 @@ class QdrantSemanticCache(BaseCache): embedding_model="text-embedding-ada-002", host_type=None, vector_size=None, - allow_legacy_unscoped_cache_hits: Optional[bool] = None, ): from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -62,16 +58,6 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model - self.allow_legacy_unscoped_cache_hits = ( - self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) - ) - if self.allow_legacy_unscoped_cache_hits: - print_verbose( - "Qdrant semantic-cache legacy unscoped hits are enabled via " - f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; searches may return " - "pre-isolation cache entries without cache-key payloads. Disable " - "this after warming the key-scoped semantic cache." - ) self.vector_size = ( vector_size if vector_size is not None else QDRANT_VECTOR_SIZE ) @@ -180,14 +166,6 @@ class QdrantSemanticCache(BaseCache): else: raise Exception("Error while creating new collection") - @classmethod - def _get_allow_legacy_unscoped_cache_hits( - cls, allow_legacy_unscoped_cache_hits: Optional[bool] - ) -> bool: - if allow_legacy_unscoped_cache_hits is not None: - return allow_legacy_unscoped_cache_hits - return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" - def _get_cache_logic(self, cached_response: Any): if cached_response is None: return cached_response @@ -210,8 +188,6 @@ class QdrantSemanticCache(BaseCache): } def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None: - if getattr(self, "allow_legacy_unscoped_cache_hits", False): - return data["filter"] = self._get_qdrant_cache_key_filter(key) def _ensure_cache_key_payload_index(self) -> None: @@ -236,14 +212,11 @@ class QdrantSemanticCache(BaseCache): ) def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: - # Legacy Qdrant semantic-cache points stored only prompt text and - # response. They cannot be reassigned to the generated LiteLLM cache key - # without risking cross-scope hits, so secure mode treats them as misses. + # Pre-isolation points stored only prompt + response with no cache-key + # payload field. Reassigning them to a caller's key would risk + # cross-scope hits, so they're treated as misses and re-populated on + # the next set_cache. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) - if cached_key is None and getattr( - self, "allow_legacy_unscoped_cache_hits", False - ): - return True return cached_key is not None and str(cached_key) == str(key) async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 35fed977cc..da9e7b1e58 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -36,9 +36,6 @@ class RedisSemanticCache(BaseCache): DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" - ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR: str = ( - "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" - ) def __init__( self, @@ -49,7 +46,6 @@ class RedisSemanticCache(BaseCache): similarity_threshold: Optional[float] = None, embedding_model: str = "text-embedding-ada-002", index_name: Optional[str] = None, - allow_legacy_unscoped_cache_hits: Optional[bool] = None, **kwargs, ): """ @@ -91,10 +87,6 @@ class RedisSemanticCache(BaseCache): # While similarity: 1 = most similar, 0 = least similar self.distance_threshold = 1 - similarity_threshold self.embedding_model = embedding_model - self.allow_legacy_unscoped_cache_hits = ( - self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) - ) - self._using_legacy_unscoped_index = False # Set up Redis connection if redis_url is None: @@ -125,14 +117,6 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) - @classmethod - def _get_allow_legacy_unscoped_cache_hits( - cls, allow_legacy_unscoped_cache_hits: Optional[bool] - ) -> bool: - if allow_legacy_unscoped_cache_hits is not None: - return allow_legacy_unscoped_cache_hits - return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" - @classmethod def _cache_key_filterable_field(cls) -> Dict[str, str]: return { @@ -167,22 +151,6 @@ class RedisSemanticCache(BaseCache): if not _is_schema_mismatch(exc): raise - if self.allow_legacy_unscoped_cache_hits: - self._using_legacy_unscoped_index = True - print_verbose( - "Redis semantic-cache legacy unscoped hits are enabled via " - f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; reusing existing " - "index without cache-key isolation. Disable this after warming " - "the isolated semantic cache." - ) - return semantic_cache_cls( - name=index_name, - redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - overwrite=False, - ) - isolated_index_name = f"{index_name}_isolated" print_verbose( "Redis semantic-cache existing index schema is not isolated; " @@ -223,11 +191,11 @@ class RedisSemanticCache(BaseCache): return Tag(self.CACHE_KEY_FIELD_NAME) == str(key) def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool: + # Pre-isolation entries with no ``litellm_cache_key`` field cannot be + # safely reassigned to a caller's scope and are treated as misses. cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) if isinstance(cached_key, bytes): cached_key = cached_key.decode("utf-8") - if cached_key is None and getattr(self, "_using_legacy_unscoped_index", False): - return True return cached_key is not None and str(cached_key) == str(key) def _get_ttl(self, **kwargs) -> Optional[int]: @@ -319,9 +287,9 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) - store_kwargs: Dict[str, Any] = {} - if not getattr(self, "_using_legacy_unscoped_index", False): - store_kwargs["filters"] = self._get_cache_filters(key) + store_kwargs: Dict[str, Any] = { + "filters": self._get_cache_filters(key), + } # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -357,11 +325,10 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - check_kwargs: Dict[str, Any] = {"prompt": prompt} - if not getattr(self, "_using_legacy_unscoped_index", False): - check_kwargs["filter_expression"] = ( - self._get_cache_key_filter_expression(key) - ) + check_kwargs: Dict[str, Any] = { + "prompt": prompt, + "filter_expression": self._get_cache_key_filter_expression(key), + } results = self.llmcache.check(**check_kwargs) # Return None if no similar prompts found @@ -475,9 +442,8 @@ class RedisSemanticCache(BaseCache): store_kwargs: Dict[str, Any] = { "vector": prompt_embedding, + "filters": self._get_cache_filters(key), } - if not getattr(self, "_using_legacy_unscoped_index", False): - store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -522,11 +488,8 @@ class RedisSemanticCache(BaseCache): check_kwargs: Dict[str, Any] = { "prompt": prompt, "vector": prompt_embedding, + "filter_expression": self._get_cache_key_filter_expression(key), } - if not getattr(self, "_using_legacy_unscoped_index", False): - check_kwargs["filter_expression"] = ( - self._get_cache_key_filter_expression(key) - ) results = await self.llmcache.acheck(**check_kwargs) # handle results / cache hit diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 9b987a6d4f..949e6ccc29 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -208,76 +208,6 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert metadata["semantic-similarity"] == 0.0 -def test_qdrant_semantic_cache_allows_legacy_unscoped_hit_with_flag(monkeypatch): - monkeypatch.setenv("LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS", "true") - - with ( - patch( - "litellm.llms.custom_httpx.http_handler._get_httpx_client" - ) as mock_sync_client, - patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), - ): - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"result": {"exists": True}} - - mock_sync_client_instance = MagicMock() - mock_sync_client_instance.get.return_value = mock_response - mock_sync_client.return_value = mock_sync_client_instance - - from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache - - qdrant_cache = QdrantSemanticCache( - collection_name="test_collection", - qdrant_api_base="http://test.qdrant.local", - qdrant_api_key="test_key", - similarity_threshold=0.8, - ) - - mock_search_response = MagicMock() - mock_search_response.status_code = 200 - mock_search_response.json.return_value = { - "result": [ - { - "payload": { - "text": "What is the capital of France?", - "response": '{"id": "test-123"}', - }, - "score": 0.9, - } - ] - } - qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) - - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} - ): - metadata = {} - result = qdrant_cache.get_cache( - key="test_key", - messages=[{"content": "What is the capital of France?"}], - metadata=metadata, - ) - - assert result == {"id": "test-123"} - assert metadata["semantic-similarity"] == 0.9 - assert "filter" not in qdrant_cache.sync_client.post.call_args.kwargs["json"] - - -def test_qdrant_semantic_cache_legacy_mode_rejects_wrong_key_hit(): - from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache - - qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) - qdrant_cache.allow_legacy_unscoped_cache_hits = True - - assert qdrant_cache._payload_matches_cache_key(payload={}, key="test_key") - assert not qdrant_cache._payload_matches_cache_key( - payload={QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "other_key"}, - key="test_key", - ) - - def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index bebe1f757b..b50a35ef50 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -232,46 +232,6 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] -def test_redis_semantic_cache_can_reuse_legacy_unscoped_index(monkeypatch): - fallback_cache_mock = MagicMock() - semantic_cache_mock = MagicMock( - side_effect=[ - ValueError("Existing index schema does not match"), - fallback_cache_mock, - ] - ) - custom_vectorizer_mock = MagicMock() - - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - monkeypatch.setenv("REDIS_HOST", "localhost") - monkeypatch.setenv("REDIS_PORT", "6379") - monkeypatch.setenv("REDIS_PASSWORD", "test_password") - monkeypatch.setenv( - RedisSemanticCache.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "true" - ) - - redis_semantic_cache = RedisSemanticCache( - similarity_threshold=0.8, - index_name="existing_index", - ) - - assert redis_semantic_cache.llmcache is fallback_cache_mock - assert redis_semantic_cache._using_legacy_unscoped_index is True - assert semantic_cache_mock.call_count == 2 - assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index" - assert "filterable_fields" not in semantic_cache_mock.call_args_list[1].kwargs - - def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( @@ -372,11 +332,12 @@ def test_redis_semantic_cache_matches_bytes_cache_key(): ) -def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): +def test_redis_semantic_cache_rejects_pre_isolation_unscoped_hit(): + """Pre-isolation entries with no cache-key field cannot be safely + reassigned to a caller's scope and are treated as misses.""" from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) - redis_semantic_cache._using_legacy_unscoped_index = False cache_hit = { "prompt": "What is the capital of France?", @@ -388,12 +349,6 @@ def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): key="test_key", ) - redis_semantic_cache._using_legacy_unscoped_index = True - assert redis_semantic_cache._cache_hit_matches_key( - cache_hit=cache_hit, - key="test_key", - ) - def test_redis_semantic_cache_builds_filter_expression(monkeypatch): class FakeTag: