From 7bda5c7cac56a206234016ba7f01317253b53ce6 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 10:59:49 -0700 Subject: [PATCH 01/17] chore(caching): isolate semantic cache entries --- litellm/caching/qdrant_semantic_cache.py | 151 +++++++------ litellm/caching/redis_semantic_cache.py | 116 ++++++++-- .../caching/test_qdrant_semantic_cache.py | 113 ++++++++-- .../caching/test_redis_semantic_cache.py | 207 +++++++++++++++++- 4 files changed, 468 insertions(+), 119 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 5e3713e5a1..b856423cbb 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -16,12 +16,17 @@ from typing import Any, cast import litellm from litellm._logging import print_verbose from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_str_from_messages, +) from litellm.types.utils import EmbeddingResponse from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): + CACHE_KEY_FIELD_NAME = "litellm_cache_key" + def __init__( # noqa: PLR0915 self, qdrant_api_base=None, @@ -170,15 +175,66 @@ class QdrantSemanticCache(BaseCache): cached_response = ast.literal_eval(cached_response) return cached_response + def _get_qdrant_cache_key_filter(self, key: str) -> dict: + return { + "must": [ + { + "key": self.CACHE_KEY_FIELD_NAME, + "match": {"value": str(key)}, + } + ] + } + + def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: + cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) + return cached_key is not None and str(cached_key) == str(key) + + async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: + llm_model_list = None + llm_router = None + + try: + from litellm.proxy.proxy_server import ( + llm_model_list as proxy_llm_model_list, + llm_router as proxy_llm_router, + ) + + llm_model_list = proxy_llm_model_list + llm_router = proxy_llm_router + except ImportError: + pass + + router_model_names = ( + [m["model_name"] for m in llm_model_list] + if llm_model_list is not None + else [] + ) + if llm_router is not None and self.embedding_model in router_model_names: + user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") + return await llm_router.aembedding( + model=self.embedding_model, + input=prompt, + cache={"no-store": True, "no-cache": True}, + metadata={ + "user_api_key": user_api_key, + "semantic-cache-embedding": True, + "trace_id": kwargs.get("metadata", {}).get("trace_id", None), + }, + ) + + return await litellm.aembedding( + model=self.embedding_model, + input=prompt, + cache={"no-store": True, "no-cache": True}, + ) + def set_cache(self, key, value, **kwargs): print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}") from litellm._uuid import uuid # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) # create an embedding for prompt embedding_response = cast( @@ -202,6 +258,7 @@ class QdrantSemanticCache(BaseCache): "id": str(uuid.uuid4()), "vector": embedding, "payload": { + self.CACHE_KEY_FIELD_NAME: str(key), "text": prompt, "response": value, }, @@ -220,9 +277,7 @@ class QdrantSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) # convert to embedding embedding_response = cast( @@ -248,6 +303,7 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, + "filter": self._get_qdrant_cache_key_filter(key), } search_response = self.sync_client.post( @@ -264,7 +320,12 @@ class QdrantSemanticCache(BaseCache): return None similarity = results[0]["score"] - cached_prompt = results[0]["payload"]["text"] + payload = results[0]["payload"] + if not self._payload_matches_cache_key(payload=payload, key=key): + print_verbose("Qdrant semantic-cache hit did not match cache key scope") + return None + + cached_prompt = payload["text"] # check similarity, if more than self.similarity_threshold, return results print_verbose( @@ -272,7 +333,7 @@ class QdrantSemanticCache(BaseCache): ) if similarity >= self.similarity_threshold: # cache hit ! - cached_value = results[0]["payload"]["response"] + cached_value = payload["response"] print_verbose( f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}" ) @@ -285,40 +346,12 @@ class QdrantSemanticCache(BaseCache): async def async_set_cache(self, key, value, **kwargs): from litellm._uuid import uuid - from litellm.proxy.proxy_server import llm_model_list, llm_router - print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}") # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] - # create an embedding for prompt - router_model_names = ( - [m["model_name"] for m in llm_model_list] - if llm_model_list is not None - else [] - ) - if llm_router is not None and self.embedding_model in router_model_names: - user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") - embedding_response = await llm_router.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - metadata={ - "user_api_key": user_api_key, - "semantic-cache-embedding": True, - "trace_id": kwargs.get("metadata", {}).get("trace_id", None), - }, - ) - else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + prompt = get_str_from_messages(messages) + embedding_response = await self._get_async_embedding(prompt, **kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] @@ -332,6 +365,7 @@ class QdrantSemanticCache(BaseCache): "id": str(uuid.uuid4()), "vector": embedding, "payload": { + self.CACHE_KEY_FIELD_NAME: str(key), "text": prompt, "response": value, }, @@ -348,38 +382,12 @@ class QdrantSemanticCache(BaseCache): async def async_get_cache(self, key, **kwargs): print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}") - from litellm.proxy.proxy_server import llm_model_list, llm_router # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = get_str_from_messages(messages) - router_model_names = ( - [m["model_name"] for m in llm_model_list] - if llm_model_list is not None - else [] - ) - if llm_router is not None and self.embedding_model in router_model_names: - user_api_key = kwargs.get("metadata", {}).get("user_api_key", "") - embedding_response = await llm_router.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - metadata={ - "user_api_key": user_api_key, - "semantic-cache-embedding": True, - "trace_id": kwargs.get("metadata", {}).get("trace_id", None), - }, - ) - else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + embedding_response = await self._get_async_embedding(prompt, **kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] @@ -395,6 +403,7 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, + "filter": self._get_qdrant_cache_key_filter(key), } search_response = await self.async_client.post( @@ -414,7 +423,13 @@ class QdrantSemanticCache(BaseCache): return None similarity = results[0]["score"] - cached_prompt = results[0]["payload"]["text"] + payload = results[0]["payload"] + if not self._payload_matches_cache_key(payload=payload, key=key): + print_verbose("Qdrant semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 + return None + + cached_prompt = payload["text"] # check similarity, if more than self.similarity_threshold, return results print_verbose( @@ -426,7 +441,7 @@ class QdrantSemanticCache(BaseCache): if similarity >= self.similarity_threshold: # cache hit ! - cached_value = results[0]["payload"]["response"] + cached_value = payload["response"] print_verbose( f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}" ) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index c76f27377d..06e9b4d0fa 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -35,6 +35,11 @@ class RedisSemanticCache(BaseCache): """ DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" + CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" + CACHE_KEY_FILTERABLE_FIELD: Dict[str, str] = { + "name": CACHE_KEY_FIELD_NAME, + "type": "tag", + } def __init__( self, @@ -66,8 +71,8 @@ class RedisSemanticCache(BaseCache): Exception: If similarity_threshold is not provided or required Redis connection information is missing """ - from redisvl.extensions.llmcache import SemanticCache - from redisvl.utils.vectorize import CustomTextVectorizer + from redisvl.extensions.llmcache import SemanticCache # type: ignore[import-not-found, import-untyped] + from redisvl.utils.vectorize import CustomTextVectorizer # type: ignore[import-not-found, import-untyped] if index_name is None: index_name = self.DEFAULT_REDIS_INDEX_NAME @@ -109,14 +114,61 @@ class RedisSemanticCache(BaseCache): # Initialize the Redis vectorizer and cache cache_vectorizer = CustomTextVectorizer(self._get_embedding) - self.llmcache = SemanticCache( - name=index_name, + self.llmcache = self._init_semantic_cache( + semantic_cache_cls=SemanticCache, + index_name=index_name, redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - overwrite=False, + cache_vectorizer=cache_vectorizer, ) + def _init_semantic_cache( + self, + semantic_cache_cls: Any, + index_name: str, + redis_url: str, + cache_vectorizer: Any, + ) -> Any: + try: + return semantic_cache_cls( + name=index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + except ValueError as exc: + if "schema does not match" not in str(exc): + raise + + isolated_index_name = f"{index_name}_isolated" + print_verbose( + "Redis semantic-cache existing index schema is not isolated; " + f"using isolated index - {isolated_index_name}" + ) + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + + def _get_cache_filters(self, key: str) -> Dict[str, str]: + return {self.CACHE_KEY_FIELD_NAME: str(key)} + + def _get_cache_key_filter_expression(self, key: str) -> Any: + from redisvl.query.filter import Tag # type: ignore[import-not-found, import-untyped] + + return Tag(self.CACHE_KEY_FIELD_NAME) == str(key) + + def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool: + cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) + if isinstance(cached_key, bytes): + cached_key = cached_key.decode("utf-8") + return cached_key is not None and str(cached_key) == str(key) + def _get_ttl(self, **kwargs) -> Optional[int]: """ Get the TTL (time-to-live) value for cache entries. @@ -188,7 +240,7 @@ class RedisSemanticCache(BaseCache): Store a value in the semantic cache. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live @@ -206,12 +258,15 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) + store_kwargs: Dict[str, Any] = {"filters": self._get_cache_filters(key)} + # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) if ttl is not None: - self.llmcache.store(prompt, value_str, ttl=int(ttl)) + store_kwargs["ttl"] = int(ttl) + self.llmcache.store(prompt, value_str, **store_kwargs) else: - self.llmcache.store(prompt, value_str) + self.llmcache.store(prompt, value_str, **store_kwargs) except Exception as e: print_verbose( f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}" @@ -222,7 +277,7 @@ class RedisSemanticCache(BaseCache): Retrieve a semantically similar cached response. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: @@ -238,8 +293,12 @@ class RedisSemanticCache(BaseCache): return None prompt = get_str_from_messages(messages) - # Check the cache for semantically similar prompts - results = self.llmcache.check(prompt=prompt) + # Check the cache for semantically similar prompts in this exact + # LiteLLM cache-key scope. + results = self.llmcache.check( + prompt=prompt, + filter_expression=self._get_cache_key_filter_expression(key), + ) # Return None if no similar prompts found if not results: @@ -247,6 +306,9 @@ class RedisSemanticCache(BaseCache): # Process the best matching result cache_hit = results[0] + if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): + print_verbose("Redis semantic-cache hit did not match cache key scope") + return None vector_distance = float(cache_hit["vector_distance"]) # Convert vector distance back to similarity score @@ -321,7 +383,7 @@ class RedisSemanticCache(BaseCache): Asynchronously store a value in the semantic cache. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live @@ -341,20 +403,25 @@ class RedisSemanticCache(BaseCache): # Generate embedding for the value (response) to cache prompt_embedding = await self._get_async_embedding(prompt, **kwargs) + store_kwargs: Dict[str, Any] = { + "vector": prompt_embedding, + "filters": self._get_cache_filters(key), + } + # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) if ttl is not None: + store_kwargs["ttl"] = ttl await self.llmcache.astore( prompt, value_str, - vector=prompt_embedding, # Pass through custom embedding - ttl=ttl, + **store_kwargs, ) else: await self.llmcache.astore( prompt, value_str, - vector=prompt_embedding, # Pass through custom embedding + **store_kwargs, ) except Exception as e: print_verbose(f"Error in async_set_cache: {str(e)}") @@ -364,7 +431,7 @@ class RedisSemanticCache(BaseCache): Asynchronously retrieve a semantically similar cached response. Args: - key: The cache key (not directly used in semantic caching) + key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: @@ -385,8 +452,13 @@ class RedisSemanticCache(BaseCache): # Generate embedding for the prompt prompt_embedding = await self._get_async_embedding(prompt, **kwargs) - # Check the cache for semantically similar prompts - results = await self.llmcache.acheck(prompt=prompt, vector=prompt_embedding) + # Check the cache for semantically similar prompts in this exact + # LiteLLM cache-key scope. + results = await self.llmcache.acheck( + prompt=prompt, + vector=prompt_embedding, + filter_expression=self._get_cache_key_filter_expression(key), + ) # handle results / cache hit if not results: @@ -396,6 +468,10 @@ class RedisSemanticCache(BaseCache): return None cache_hit = results[0] + if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): + print_verbose("Redis semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 + return None vector_distance = float(cache_hit["vector_distance"]) # Convert vector distance back to similarity diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 13dc4b5812..f357d97341 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -19,9 +19,7 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -67,9 +65,7 @@ def test_qdrant_semantic_cache_get_cache_hit(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -98,6 +94,7 @@ def test_qdrant_semantic_cache_get_cache_hit(): "result": [ { "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", "text": "What is the capital of France?", # Original prompt "response": '{"id": "test-123", "choices": [{"message": {"content": "Paris is the capital of France."}}]}', }, @@ -127,6 +124,67 @@ def test_qdrant_semantic_cache_get_cache_hit(): # Verify search was called qdrant_cache.sync_client.post.assert_called() + assert qdrant_cache.sync_client.post.call_args.kwargs["json"]["filter"] == { + "must": [ + { + "key": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "match": {"value": "test_key"}, + } + ] + } + + +def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): + """ + Test QDRANT semantic cache rejects old or unscoped cache hits. + """ + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), + ): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = { + "result": [ + { + "payload": { + "text": "What is the capital of France?", + "response": '{"id": "test-123"}', + }, + "score": 0.9, + } + ] + } + qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", messages=[{"content": "What is the capital of France?"}] + ) + + assert result is None def test_qdrant_semantic_cache_get_cache_miss(): @@ -138,9 +196,7 @@ def test_qdrant_semantic_cache_get_cache_miss(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -230,6 +286,7 @@ async def test_qdrant_semantic_cache_async_get_cache_hit(): "result": [ { "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", "text": "What is the capital of Spain?", # Original prompt "response": '{"id": "test-456", "choices": [{"message": {"content": "Madrid is the capital of Spain."}}]}', }, @@ -262,6 +319,16 @@ async def test_qdrant_semantic_cache_async_get_cache_hit(): # Verify async search was called qdrant_cache.async_client.post.assert_called() + assert qdrant_cache.async_client.post.call_args.kwargs["json"][ + "filter" + ] == { + "must": [ + { + "key": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "match": {"value": "test_key"}, + } + ] + } @pytest.mark.asyncio @@ -336,9 +403,7 @@ def test_qdrant_semantic_cache_set_cache(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -384,6 +449,12 @@ def test_qdrant_semantic_cache_set_cache(): # Verify upsert was called qdrant_cache.sync_client.put.assert_called() + upsert_payload = qdrant_cache.sync_client.put.call_args.kwargs["json"][ + "points" + ][0]["payload"] + assert ( + upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" + ) @pytest.mark.asyncio @@ -450,6 +521,12 @@ async def test_qdrant_semantic_cache_async_set_cache(): # Verify async upsert was called qdrant_cache.async_client.put.assert_called() + upsert_payload = qdrant_cache.async_client.put.call_args.kwargs["json"][ + "points" + ][0]["payload"] + assert ( + upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" + ) def test_qdrant_semantic_cache_custom_vector_size(): @@ -462,9 +539,7 @@ def test_qdrant_semantic_cache_custom_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection does NOT exist (so it will be created) @@ -521,9 +596,7 @@ def test_qdrant_semantic_cache_default_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection exists check @@ -559,9 +632,7 @@ def test_qdrant_semantic_cache_large_vector_size(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), ): # Mock the collection does NOT exist (so it will be created) diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index f9946e266f..9f2f390f4e 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -72,13 +72,22 @@ def test_redis_semantic_cache_get_cache(monkeypatch): "prompt": "What is the capital of France?", "response": '{"content": "Paris is the capital of France."}', "vector_distance": 0.1, # Distance of 0.1 means similarity of 0.9 + RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", } ] redis_semantic_cache.llmcache.check = MagicMock(return_value=mock_result) # Mock the embedding function - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + with ( + patch( + "litellm.embedding", + return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]}, + ), + patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ), ): # Test get_cache with a message result = redis_semantic_cache.get_cache( @@ -89,7 +98,131 @@ def test_redis_semantic_cache_get_cache(monkeypatch): assert result == {"content": "Paris is the capital of France."} # Verify llmcache.check was called - redis_semantic_cache.llmcache.check.assert_called_once() + redis_semantic_cache.llmcache.check.assert_called_once_with( + prompt="What is the capital of France?", + filter_expression="cache-key-filter", + ) + + +def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.check = MagicMock( + return_value=[ + { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + ] + ) + + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + result = redis_semantic_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + ) + + assert result is None + + +def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.store = MagicMock() + + redis_semantic_cache.set_cache( + key="test_key", + value={"content": "Paris"}, + messages=[{"content": "What is the capital of France?"}], + ttl=60, + ) + + redis_semantic_cache.llmcache.store.assert_called_once_with( + "What is the capital of France?", + "{'content': 'Paris'}", + filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, + ttl=60, + ) + + +def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert semantic_cache_mock.call_args_list[0].kwargs["name"] == "existing_index" + assert ( + semantic_cache_mock.call_args_list[1].kwargs["name"] + == "existing_index_isolated" + ) + assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ + RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + ] @pytest.mark.asyncio @@ -123,6 +256,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): "prompt": "What is the capital of France?", "response": '{"content": "Paris is the capital of France."}', "vector_distance": 0.1, # Distance of 0.1 means similarity of 0.9 + RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", } ] @@ -131,16 +265,69 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): return_value=[0.1, 0.2, 0.3] ) - # Test async_get_cache with a message - result = await redis_semantic_cache.async_get_cache( - key="test_key", - messages=[{"content": "What is the capital of France?"}], - metadata={}, - ) + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + # Test async_get_cache with a message + result = await redis_semantic_cache.async_get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata={}, + ) # Verify result is properly parsed assert result == {"content": "Paris is the capital of France."} # Verify methods were called redis_semantic_cache._get_async_embedding.assert_called_once() - redis_semantic_cache.llmcache.acheck.assert_called_once() + redis_semantic_cache.llmcache.acheck.assert_called_once_with( + prompt="What is the capital of France?", + vector=[0.1, 0.2, 0.3], + filter_expression="cache-key-filter", + ) + + +@pytest.mark.asyncio +async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( + monkeypatch, +): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.astore = AsyncMock() + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) + + await redis_semantic_cache.async_set_cache( + key="test_key", + value={"content": "Paris"}, + messages=[{"content": "What is the capital of France?"}], + ttl=60, + ) + + redis_semantic_cache.llmcache.astore.assert_called_once_with( + "What is the capital of France?", + "{'content': 'Paris'}", + vector=[0.1, 0.2, 0.3], + filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, + ttl=60, + ) From 9aa3dfc816f5472dc9ef28854c6198109b1ada69 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 01:10:44 -0700 Subject: [PATCH 02/17] chore(proxy): stabilize lazy openapi snapshot --- litellm/proxy/_lazy_openapi_snapshot.json | 28 ++++++------- litellm/proxy/_lazy_openapi_snapshot.py | 40 ++++++++++++++++++- .../proxy/test_lazy_openapi_snapshot.py | 35 ++++++++++++++++ 3 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 tests/test_litellm/proxy/test_lazy_openapi_snapshot.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8331f748c6..46a514c087 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3572,7 +3572,7 @@ "/anthropic/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__delete", "parameters": [ { "in": "path", @@ -3616,7 +3616,7 @@ }, "get": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__get", "parameters": [ { "in": "path", @@ -3660,7 +3660,7 @@ }, "patch": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__patch", "parameters": [ { "in": "path", @@ -3704,7 +3704,7 @@ }, "post": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", - "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "operationId": "anthropic_proxy_route_anthropic__endpoint__post", "parameters": [ { "in": "path", @@ -13260,7 +13260,7 @@ "/langfuse/{endpoint}": { "delete": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__delete", "parameters": [ { "in": "path", @@ -13299,7 +13299,7 @@ }, "get": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__get", "parameters": [ { "in": "path", @@ -13338,7 +13338,7 @@ }, "patch": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__patch", "parameters": [ { "in": "path", @@ -13377,7 +13377,7 @@ }, "post": { "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", - "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "operationId": "langfuse_proxy_route_langfuse__endpoint__post", "parameters": [ { "in": "path", @@ -26883,7 +26883,7 @@ "/toolset/{toolset_name}/mcp": { "delete": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete", "parameters": [ { "in": "path", @@ -26922,7 +26922,7 @@ }, "get": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get", "parameters": [ { "in": "path", @@ -26961,7 +26961,7 @@ }, "head": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head", "parameters": [ { "in": "path", @@ -27000,7 +27000,7 @@ }, "options": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options", "parameters": [ { "in": "path", @@ -27039,7 +27039,7 @@ }, "patch": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch", "parameters": [ { "in": "path", @@ -27078,7 +27078,7 @@ }, "post": { "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", - "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post", "parameters": [ { "in": "path", diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 315f6a9742..309a0276aa 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Dict, Optional SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json" +HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"} def load_snapshot() -> Optional[Dict[str, Dict]]: @@ -25,6 +26,39 @@ def load_snapshot() -> Optional[Dict[str, Dict]]: return None +def _normalize_operation_ids(paths: Dict[str, Dict]) -> None: + """Make FastAPI-generated operation IDs stable for multi-method routes. + + FastAPI derives the default operation ID suffix from the first item in the + route's methods set. For routes registered with several HTTP methods, that + set iteration order can vary between processes, which makes the snapshot + drift even when no routes changed. + """ + for path_ops in paths.values(): + if not isinstance(path_ops, dict): + continue + + methods = {method for method in path_ops if method in HTTP_METHODS} + if not methods: + continue + + for method, operation in path_ops.items(): + if method not in HTTP_METHODS or not isinstance(operation, dict): + continue + + operation_id = operation.get("operationId") + if not isinstance(operation_id, str): + continue + + for suffix in methods: + suffix_token = f"_{suffix}" + if operation_id.endswith(suffix_token): + operation["operationId"] = ( + operation_id[: -len(suffix_token)] + f"_{method}" + ) + break + + def generate_snapshot() -> Dict[str, Dict]: import importlib @@ -52,13 +86,15 @@ def generate_snapshot() -> Dict[str, Dict]: if not feat_routes: continue full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + paths = full.get("paths", {}) + _normalize_operation_ids(paths) # Group all of a feature's routes under one tag. - for path_ops in full.get("paths", {}).values(): + for path_ops in paths.values(): for op in path_ops.values(): if isinstance(op, dict): op["tags"] = [feat.name] fragments[feat.name] = { - "paths": full.get("paths", {}), + "paths": paths, "components": {"schemas": full.get("components", {}).get("schemas", {})}, } return fragments diff --git a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py new file mode 100644 index 0000000000..8bc39c93ee --- /dev/null +++ b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py @@ -0,0 +1,35 @@ +from litellm.proxy._lazy_openapi_snapshot import _normalize_operation_ids + + +def test_normalize_operation_ids_uses_each_http_method(): + paths = { + "/proxy/{endpoint}": { + "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, + "get": {"operationId": "proxy_route_proxy__endpoint__put"}, + "post": {"operationId": "proxy_route_proxy__endpoint__put"}, + "put": {"operationId": "proxy_route_proxy__endpoint__put"}, + } + } + + _normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" + assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" + assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" + assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" + + +def test_normalize_operation_ids_preserves_custom_ids(): + paths = { + "/proxy/{endpoint}": { + "get": {"operationId": "custom_operation"}, + "post": {"operationId": "custom_operation"}, + } + } + + _normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["get"]["operationId"] == "custom_operation" + assert operations["post"]["operationId"] == "custom_operation" From ae9b63c468bcca396ec0b1e82a93df0fae86e065 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:15:52 -0700 Subject: [PATCH 03/17] chore(caching): index qdrant semantic cache scope --- litellm/caching/qdrant_semantic_cache.py | 26 ++++++++++++++ .../caching/test_qdrant_semantic_cache.py | 36 ++++++++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index b856423cbb..e6e8e25522 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -120,6 +120,7 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"Collection already exists.\nCollection details:{self.collection_info}" ) + self._ensure_cache_key_payload_index() else: if quantization_config is None or quantization_config == "binary": quantization_params = { @@ -161,6 +162,7 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"New collection created.\nCollection details:{self.collection_info}" ) + self._ensure_cache_key_payload_index() else: raise Exception("Error while creating new collection") @@ -185,7 +187,31 @@ class QdrantSemanticCache(BaseCache): ] } + def _ensure_cache_key_payload_index(self) -> None: + try: + response = self.sync_client.put( + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/index", + headers=self.headers, + json={ + "field_name": self.CACHE_KEY_FIELD_NAME, + "field_schema": "keyword", + }, + ) + if response.status_code not in (200, 201): + print_verbose( + "Qdrant semantic-cache could not create cache-key payload index: " + f"{response.text}" + ) + except Exception as exc: + print_verbose( + "Qdrant semantic-cache could not create cache-key payload index: " + f"{str(exc)}" + ) + def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: + # Legacy Qdrant semantic-cache points stored only prompt text and + # response. They cannot be reassigned to the generated LiteLLM cache key + # without risking cross-scope hits, so they must be treated as misses. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) return cached_key is not None and str(cached_key) == str(key) diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index f357d97341..733cf47c89 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -29,6 +29,9 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): mock_sync_client_instance = MagicMock() mock_sync_client_instance.get.return_value = mock_response + mock_index_response = MagicMock() + mock_index_response.status_code = 200 + mock_sync_client_instance.put.return_value = mock_index_response mock_sync_client.return_value = mock_sync_client_instance from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache @@ -46,6 +49,17 @@ def test_qdrant_semantic_cache_initialization(monkeypatch): assert qdrant_cache.qdrant_api_base == "http://test.qdrant.local" assert qdrant_cache.qdrant_api_key == "test_key" assert qdrant_cache.similarity_threshold == 0.8 + mock_sync_client_instance.put.assert_called_once_with( + url="http://test.qdrant.local/collections/test_collection/index", + headers={ + "Content-Type": "application/json", + "api-key": "test_key", + }, + json={ + "field_name": QdrantSemanticCache.CACHE_KEY_FIELD_NAME, + "field_schema": "keyword", + }, + ) # Test initialization with missing similarity_threshold with pytest.raises(Exception, match="similarity_threshold must be provided"): @@ -137,6 +151,9 @@ def test_qdrant_semantic_cache_get_cache_hit(): def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): """ Test QDRANT semantic cache rejects old or unscoped cache hits. + + Legacy points have only text and response payloads, so they cannot be + safely migrated to a generated LiteLLM cache key. """ with ( patch( @@ -580,9 +597,13 @@ def test_qdrant_semantic_cache_custom_vector_size(): assert qdrant_cache.vector_size == 768 # Verify the PUT call to create the collection used vector_size=768 - put_call = mock_sync_client_instance.put.call_args - assert put_call is not None - create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + put_call = next( + call + for call in mock_sync_client_instance.put.call_args_list + if call.kwargs["url"] + == "http://test.qdrant.local/collections/test_collection_768" + ) + create_payload = put_call.kwargs["json"] assert create_payload["vectors"]["size"] == 768 assert create_payload["vectors"]["distance"] == "Cosine" @@ -670,6 +691,11 @@ def test_qdrant_semantic_cache_large_vector_size(): assert qdrant_cache.vector_size == 4096 # Verify the collection was created with 4096 - put_call = mock_sync_client_instance.put.call_args - create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + put_call = next( + call + for call in mock_sync_client_instance.put.call_args_list + if call.kwargs["url"] + == "http://test.qdrant.local/collections/test_collection_4096" + ) + create_payload = put_call.kwargs["json"] assert create_payload["vectors"]["size"] == 4096 From 1c19bdda79447249e81f9530e072941c577e844f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:26:03 -0700 Subject: [PATCH 04/17] test(caching): cover semantic cache isolation guards --- litellm/proxy/_lazy_openapi_snapshot.py | 4 +- .../caching/test_qdrant_semantic_cache.py | 33 +++++++ .../caching/test_redis_semantic_cache.py | 94 +++++++++++++++++++ .../proxy/test_lazy_openapi_snapshot.py | 35 ------- .../test_lazy_openapi_snapshot.py | 76 +++++++++++++++ 5 files changed, 205 insertions(+), 37 deletions(-) delete mode 100644 tests/test_litellm/proxy/test_lazy_openapi_snapshot.py create mode 100644 tests/test_litellm/test_lazy_openapi_snapshot.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 309a0276aa..eb32443baa 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -59,7 +59,7 @@ def _normalize_operation_ids(paths: Dict[str, Dict]) -> None: break -def generate_snapshot() -> Dict[str, Dict]: +def generate_snapshot() -> Dict[str, Dict]: # pragma: no cover import importlib from fastapi.openapi.utils import get_openapi @@ -100,7 +100,7 @@ def generate_snapshot() -> Dict[str, Dict]: return fragments -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover fragments = generate_snapshot() SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n") sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n") diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 733cf47c89..aa46d18023 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -204,6 +204,39 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert result is None +def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = {"Content-Type": "application/json"} + qdrant_cache.sync_client = MagicMock() + response = MagicMock() + response.status_code = 400 + response.text = "bad index" + qdrant_cache.sync_client.put.return_value = response + + qdrant_cache._ensure_cache_key_payload_index() + + qdrant_cache.sync_client.put.assert_called_once() + + +def test_qdrant_semantic_cache_payload_index_exception_is_non_blocking(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = {"Content-Type": "application/json"} + qdrant_cache.sync_client = MagicMock() + qdrant_cache.sync_client.put.side_effect = Exception("boom") + + qdrant_cache._ensure_cache_key_payload_index() + + qdrant_cache.sync_client.put.assert_called_once() + + def test_qdrant_semantic_cache_get_cache_miss(): """ Test QDRANT semantic cache get method when there's a cache miss. diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 9f2f390f4e..b702de8d5f 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -225,6 +225,52 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] +def test_redis_semantic_cache_reraises_unexpected_index_error(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache.distance_threshold = 0.2 + semantic_cache_mock = MagicMock(side_effect=ValueError("connection failed")) + + with pytest.raises(ValueError, match="connection failed"): + redis_semantic_cache._init_semantic_cache( + semantic_cache_cls=semantic_cache_mock, + index_name="existing_index", + redis_url="redis://localhost:6379", + cache_vectorizer=MagicMock(), + ) + + +def test_redis_semantic_cache_matches_bytes_cache_key(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + + assert redis_semantic_cache._cache_hit_matches_key( + cache_hit={RedisSemanticCache.CACHE_KEY_FIELD_NAME: b"test_key"}, + key="test_key", + ) + + +def test_redis_semantic_cache_builds_filter_expression(monkeypatch): + class FakeTag: + def __init__(self, field_name): + self.field_name = field_name + + def __eq__(self, value): + return (self.field_name, value) + + with patch.dict("sys.modules", {"redisvl.query.filter": MagicMock(Tag=FakeTag)}): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + + assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( + RedisSemanticCache.CACHE_KEY_FIELD_NAME, + "test_key", + ) + + @pytest.mark.asyncio async def test_redis_semantic_cache_async_get_cache(monkeypatch): # Mock the redisvl import @@ -289,6 +335,54 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): ) +@pytest.mark.asyncio +async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeypatch): + semantic_cache_mock = MagicMock() + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) + redis_semantic_cache.llmcache.acheck = AsyncMock( + return_value=[ + { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + ] + ) + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) + + with patch.object( + redis_semantic_cache, + "_get_cache_key_filter_expression", + return_value="cache-key-filter", + ): + result = await redis_semantic_cache.async_get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata={}, + ) + + assert result is None + + @pytest.mark.asyncio async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( monkeypatch, diff --git a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py deleted file mode 100644 index 8bc39c93ee..0000000000 --- a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py +++ /dev/null @@ -1,35 +0,0 @@ -from litellm.proxy._lazy_openapi_snapshot import _normalize_operation_ids - - -def test_normalize_operation_ids_uses_each_http_method(): - paths = { - "/proxy/{endpoint}": { - "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, - "get": {"operationId": "proxy_route_proxy__endpoint__put"}, - "post": {"operationId": "proxy_route_proxy__endpoint__put"}, - "put": {"operationId": "proxy_route_proxy__endpoint__put"}, - } - } - - _normalize_operation_ids(paths) - - operations = paths["/proxy/{endpoint}"] - assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" - assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" - assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" - assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" - - -def test_normalize_operation_ids_preserves_custom_ids(): - paths = { - "/proxy/{endpoint}": { - "get": {"operationId": "custom_operation"}, - "post": {"operationId": "custom_operation"}, - } - } - - _normalize_operation_ids(paths) - - operations = paths["/proxy/{endpoint}"] - assert operations["get"]["operationId"] == "custom_operation" - assert operations["post"]["operationId"] == "custom_operation" diff --git a/tests/test_litellm/test_lazy_openapi_snapshot.py b/tests/test_litellm/test_lazy_openapi_snapshot.py new file mode 100644 index 0000000000..c33bab2c75 --- /dev/null +++ b/tests/test_litellm/test_lazy_openapi_snapshot.py @@ -0,0 +1,76 @@ +from litellm.proxy import _lazy_openapi_snapshot as snapshot_module + + +def test_load_snapshot_returns_none_when_missing(monkeypatch, tmp_path): + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", tmp_path / "missing.json") + + assert snapshot_module.load_snapshot() is None + + +def test_load_snapshot_reads_json(monkeypatch, tmp_path): + snapshot_file = tmp_path / "snapshot.json" + snapshot_file.write_text('{"mcp": {"paths": {}}}') + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", snapshot_file) + + assert snapshot_module.load_snapshot() == {"mcp": {"paths": {}}} + + +def test_load_snapshot_returns_none_for_invalid_json(monkeypatch, tmp_path): + snapshot_file = tmp_path / "snapshot.json" + snapshot_file.write_text("{") + monkeypatch.setattr(snapshot_module, "SNAPSHOT_FILE", snapshot_file) + + assert snapshot_module.load_snapshot() is None + + +def test_normalize_operation_ids_uses_each_http_method(): + paths = { + "/proxy/{endpoint}": { + "delete": {"operationId": "proxy_route_proxy__endpoint__put"}, + "get": {"operationId": "proxy_route_proxy__endpoint__put"}, + "post": {"operationId": "proxy_route_proxy__endpoint__put"}, + "put": {"operationId": "proxy_route_proxy__endpoint__put"}, + } + } + + snapshot_module._normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["delete"]["operationId"] == "proxy_route_proxy__endpoint__delete" + assert operations["get"]["operationId"] == "proxy_route_proxy__endpoint__get" + assert operations["post"]["operationId"] == "proxy_route_proxy__endpoint__post" + assert operations["put"]["operationId"] == "proxy_route_proxy__endpoint__put" + + +def test_normalize_operation_ids_preserves_custom_ids(): + paths = { + "/proxy/{endpoint}": { + "get": {"operationId": "custom_operation"}, + "post": {"operationId": "custom_operation"}, + } + } + + snapshot_module._normalize_operation_ids(paths) + + operations = paths["/proxy/{endpoint}"] + assert operations["get"]["operationId"] == "custom_operation" + assert operations["post"]["operationId"] == "custom_operation" + + +def test_normalize_operation_ids_skips_invalid_entries(): + paths = { + "/not-a-dict": "skip", + "/no-http-methods": {"parameters": []}, + "/invalid-operation": { + "get": ["skip"], + "post": {"operationId": 123}, + "parameters": [], + }, + } + + snapshot_module._normalize_operation_ids(paths) + + assert paths["/not-a-dict"] == "skip" + assert paths["/no-http-methods"] == {"parameters": []} + assert paths["/invalid-operation"]["get"] == ["skip"] + assert paths["/invalid-operation"]["post"] == {"operationId": 123} From 74e93444cf0b08198bcd5ee1dc1a822009211c04 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 11:57:41 -0700 Subject: [PATCH 05/17] chore(caching): align qdrant scoped miss metadata --- litellm/caching/qdrant_semantic_cache.py | 1 + tests/test_litellm/caching/test_qdrant_semantic_cache.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index e6e8e25522..e7436e2ca6 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -349,6 +349,7 @@ class QdrantSemanticCache(BaseCache): payload = results[0]["payload"] if not self._payload_matches_cache_key(payload=payload, key=key): print_verbose("Qdrant semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None cached_prompt = payload["text"] diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index aa46d18023..71dbd24063 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -197,11 +197,15 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): with patch( "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} ): + metadata = {} result = qdrant_cache.get_cache( - key="test_key", messages=[{"content": "What is the capital of France?"}] + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) assert result is None + assert metadata["semantic-similarity"] == 0.0 def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): From d8c11f962296feb8c7489bf6115062984e9716f7 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 12:10:00 -0700 Subject: [PATCH 06/17] chore(caching): align redis semantic miss metadata --- litellm/caching/redis_semantic_cache.py | 7 +++++++ tests/test_litellm/caching/test_redis_semantic_cache.py | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 06e9b4d0fa..85d3a4f1b4 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -290,6 +290,7 @@ class RedisSemanticCache(BaseCache): messages = kwargs.get("messages", []) if not messages: print_verbose("No messages provided for semantic cache lookup") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None prompt = get_str_from_messages(messages) @@ -302,12 +303,14 @@ class RedisSemanticCache(BaseCache): # Return None if no similar prompts found if not results: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None # Process the best matching result cache_hit = results[0] if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key): print_verbose("Redis semantic-cache hit did not match cache key scope") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None vector_distance = float(cache_hit["vector_distance"]) @@ -319,6 +322,9 @@ class RedisSemanticCache(BaseCache): cached_prompt = cache_hit["prompt"] cached_response = cache_hit["response"] + # update kwargs["metadata"] with similarity, don't rewrite the original metadata + kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity + print_verbose( f"Cache hit: similarity threshold: {self.similarity_threshold}, " f"actual similarity: {similarity}, " @@ -329,6 +335,7 @@ class RedisSemanticCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: print_verbose(f"Error retrieving from Redis semantic cache: {str(e)}") + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def _get_async_embedding(self, prompt: str, **kwargs) -> List[float]: """ diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index b702de8d5f..f5c84c9d3e 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -90,12 +90,16 @@ def test_redis_semantic_cache_get_cache(monkeypatch): ), ): # Test get_cache with a message + metadata = {} result = redis_semantic_cache.get_cache( - key="test_key", messages=[{"content": "What is the capital of France?"}] + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) # Verify result is properly parsed assert result == {"content": "Paris is the capital of France."} + assert metadata["semantic-similarity"] == pytest.approx(0.9) # Verify llmcache.check was called redis_semantic_cache.llmcache.check.assert_called_once_with( @@ -139,12 +143,15 @@ def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): "_get_cache_key_filter_expression", return_value="cache-key-filter", ): + metadata = {} result = redis_semantic_cache.get_cache( key="test_key", messages=[{"content": "What is the capital of France?"}], + metadata=metadata, ) assert result is None + assert metadata["semantic-similarity"] == 0.0 def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): From 8cb52ce0bbcae4d1e421796224543a4c118fa41f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Fri, 1 May 2026 22:15:03 -0700 Subject: [PATCH 07/17] fix(caching): handle stale isolated Redis semantic index --- litellm/caching/redis_semantic_cache.py | 38 +++++++--- .../caching/test_redis_semantic_cache.py | 73 +++++++++++++++++++ 2 files changed, 102 insertions(+), 9 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 85d3a4f1b4..27538530bc 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -128,6 +128,9 @@ class RedisSemanticCache(BaseCache): redis_url: str, cache_vectorizer: Any, ) -> Any: + def _is_schema_mismatch(exc: ValueError) -> bool: + return "schema does not match" in str(exc) + try: return semantic_cache_cls( name=index_name, @@ -138,7 +141,7 @@ class RedisSemanticCache(BaseCache): overwrite=False, ) except ValueError as exc: - if "schema does not match" not in str(exc): + if not _is_schema_mismatch(exc): raise isolated_index_name = f"{index_name}_isolated" @@ -146,14 +149,31 @@ class RedisSemanticCache(BaseCache): "Redis semantic-cache existing index schema is not isolated; " f"using isolated index - {isolated_index_name}" ) - return semantic_cache_cls( - name=isolated_index_name, - redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], - overwrite=False, - ) + try: + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=False, + ) + except ValueError as isolated_exc: + if not _is_schema_mismatch(isolated_exc): + raise + + print_verbose( + "Redis semantic-cache isolated index schema is stale; " + f"recreating isolated index - {isolated_index_name}" + ) + return semantic_cache_cls( + name=isolated_index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + overwrite=True, + ) def _get_cache_filters(self, key: str) -> Dict[str, str]: return {self.CACHE_KEY_FIELD_NAME: str(key)} diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index f5c84c9d3e..02b06bed62 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -232,6 +232,79 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] +def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert ( + semantic_cache_mock.call_args_list[2].kwargs["name"] + == "existing_index_isolated" + ) + assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True + assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ + RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + ] + + +def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypatch): + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + ValueError("connection failed"), + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + + with pytest.raises(ValueError, match="connection failed"): + RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + def test_redis_semantic_cache_reraises_unexpected_index_error(): from litellm.caching.redis_semantic_cache import RedisSemanticCache From 37a22acf6f527870d4873c33c67a1078dc20f52b Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Sun, 3 May 2026 09:06:29 +0000 Subject: [PATCH 08/17] chore(proxy): close callback-config and observability-credential side channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related gaps in the proxy's request bouncer: 1. ``is_request_body_safe`` (auth_utils.py) walked the request-body root and the ``litellm_embedding_config`` nested dict, but not ``metadata`` or ``litellm_metadata``. The same fields it bans at root — Langfuse / Langsmith / Arize / PostHog / Braintrust / Phoenix / W&B Weave / GCS / Humanloop / Lunary credentials and routing — were silently accepted when the caller put them inside metadata, retargeting observability callbacks to a caller-controlled host with caller-supplied creds. Walk both metadata containers (and parse the JSON-string form sent via multipart / ``extra_body``) through the same banned-params helper, so the existing ``allow_client_side_credentials`` opt-in covers both paths consistently. 2. The banned-params list was hand-maintained and lagged the canonical ``_supported_callback_params`` allow-list in ``initialize_dynamic_callback_params``. Derive the observability bans from that allow-list (minus a small ``_SAFE_CLIENT_CALLBACK_PARAMS`` set for informational fields like ``langfuse_prompt_version`` and ``langsmith_sampling_rate``) so future integrations are covered automatically; ``_EXTRA_BANNED_OBSERVABILITY_PARAMS`` carries the handful of fields integrations read but the allow-list hasn't caught up to. A guard test fails CI if a new entry is added to ``_supported_callback_params`` without an explicit safe-list decision. Separately in ``litellm_pre_call_utils.py``: add ``callbacks``, ``service_callback``, ``logger_fn``, and ``litellm_disabled_callbacks`` to ``_UNTRUSTED_ROOT_CONTROL_FIELDS``. The first three are appended to worker-wide ``litellm.{input,success,failure,_async_*,service}_callback`` lists / ``litellm.user_logger_fn`` from inside ``function_setup`` — one request poisons every subsequent caller in that worker. The last is the inverse primitive: the legitimate path reads it from key/team metadata, the request-body version silently disables admin-configured audit / observability for the call. Co-Authored-By: Claude Opus 4.7 (1M context) --- litellm/proxy/auth/auth_utils.py | 100 +++++++++- litellm/proxy/litellm_pre_call_utils.py | 13 ++ .../proxy/auth/test_auth_utils.py | 186 ++++++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 53 +++++ 4 files changed, 343 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 51108827f6..84fb2f8a95 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2,7 +2,7 @@ import os import re import sys from functools import lru_cache -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Tuple, Union from fastapi import HTTPException, Request, status @@ -173,9 +173,67 @@ def _allow_model_level_clientside_configurable_parameters( # threat shape should be added here. _NESTED_CONFIG_KEYS: Tuple[str, ...] = ("litellm_embedding_config",) -# Banned root-level params. Same list applies to every entry in -# ``_NESTED_CONFIG_KEYS`` because those dicts get spread as ``**kwargs`` -# into the same outbound calls. +# Metadata containers that carry per-request configuration consumed by the +# observability callbacks. The same banned-param list applies — a value +# under ``metadata.langfuse_host`` redirects the same Langfuse client and +# leaks the same credentials as the root-level ``langfuse_host``, but the +# original check only walked the request-body root, so the metadata path +# was an unintentional bypass. +_NESTED_METADATA_KEYS: Tuple[str, ...] = ("metadata", "litellm_metadata") + +# Banned request-body params. The same list applies to every entry in +# ``_NESTED_CONFIG_KEYS`` (dicts spread as ``**kwargs`` into outbound +# calls) and ``_NESTED_METADATA_KEYS`` (dicts read directly by integration +# callbacks), so a single banned name is enforced wherever the field can +# reach the call path from. +# Per-request observability params that are SAFE to accept from clients. +# These describe the request being logged (prompt version, sampling rate) +# without choosing the destination or the credentials, so they don't +# contribute to the data-exfil primitive that the rest of +# ``_supported_callback_params`` does. +_SAFE_CLIENT_CALLBACK_PARAMS: FrozenSet[str] = frozenset( + { + "langfuse_prompt_version", + "langsmith_sampling_rate", + } +) + +# Observability fields that integrations read from the request body or +# metadata but that are not (yet) listed in ``_supported_callback_params``. +# Listed here so the proxy bans them today; the long-term cleanup is to +# fold these into the canonical allowlist so they share one source of +# truth with the rest. +_EXTRA_BANNED_OBSERVABILITY_PARAMS: FrozenSet[str] = frozenset( + { + "posthog_api_url", + "phoenix_project_name", + "wandb_api_key", + "weave_project_id", + } +) + + +def _build_banned_observability_params() -> FrozenSet[str]: + """Derive the observability ban list from the canonical allowlist. + + ``_supported_callback_params`` in + ``litellm/litellm_core_utils/initialize_dynamic_callback_params.py`` is + the single place that enumerates every observability field + integrations resolve from kwargs/metadata. Subtract the small set of + informational fields (``_SAFE_CLIENT_CALLBACK_PARAMS``) and union with + the extras the canonical allowlist hasn't caught up to yet. New + integrations added to the canonical allowlist are banned by default, + which is the safe failure mode. + """ + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + _supported_callback_params, + ) + + return ( + frozenset(_supported_callback_params) - _SAFE_CLIENT_CALLBACK_PARAMS + ) | _EXTRA_BANNED_OBSERVABILITY_PARAMS + + _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( "api_base", "base_url", @@ -190,11 +248,6 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( # tokens) to the attacker's host, or coerces the proxy into # authenticating against the attacker's host with admin secrets. "aws_bedrock_runtime_endpoint", - "langsmith_base_url", - "langfuse_host", - "posthog_host", - "braintrust_host", - "slack_webhook_url", # Provider-specific endpoint overrides that flow into the outbound # request via ``optional_params``. Same threat as ``api_base``: # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker @@ -203,6 +256,11 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( "s3_endpoint_url", "sagemaker_base_url", "deployment_url", + # Observability credentials, hosts, and project identifiers: derived + # from the canonical ``_supported_callback_params`` allowlist so new + # integrations are covered automatically. Sorted for stable iteration + # order and reviewable diffs. + *sorted(_build_banned_observability_params()), ) @@ -275,9 +333,33 @@ def is_request_body_safe( nested = request_body.get(nested_key) if isinstance(nested, dict): _check_banned_params(nested, general_settings, llm_router, model) + for metadata_key in _NESTED_METADATA_KEYS: + metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) + if metadata is not None: + _check_banned_params(metadata, general_settings, llm_router, model) return True +def _coerce_metadata_to_dict(value: Any) -> Optional[Dict[str, Any]]: + """Return ``value`` as a dict, parsing it from JSON if delivered as a string. + + Multipart/form-data and ``extra_body`` callers send ``litellm_metadata`` + as a JSON-encoded string; the proxy parses it into a dict later in + ``add_litellm_data_to_request``, but the auth-time bouncer runs first + and would otherwise miss the banned-param check on a still-stringified + metadata blob. + """ + if isinstance(value, dict): + return value + if isinstance(value, str): + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads + + parsed = safe_json_loads(value) + if isinstance(parsed, dict): + return parsed + return None + + async def pre_db_read_auth_checks( request: Request, request_data: dict, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 853c56856f..c1d133b032 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -120,6 +120,19 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS = ( "pillar_response_headers", "_guardrail_pipelines", "_pipeline_managed_guardrails", + # Callback-registration fields. ``callbacks``, ``service_callback``, + # and ``logger_fn`` are read by ``litellm.utils.function_setup`` and + # appended to process-wide ``litellm.{input,success,failure,_async_*, + # service}_callback`` lists / ``litellm.user_logger_fn`` — one request + # poisons the worker for every subsequent caller. + # ``litellm_disabled_callbacks`` is the inverse primitive: the + # legitimate path reads it from key/team metadata, the request-body + # version silently turns off admin-configured audit/observability + # for the caller's request. + "callbacks", + "service_callback", + "logger_fn", + "litellm_disabled_callbacks", ) _UNTRUSTED_METADATA_CONTROL_FIELDS = ( diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index c146b5ded5..58a55f8aa6 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1292,3 +1292,189 @@ class TestIsRequestBodySafeNestedConfig: ) is True ) + + +# ── observability-callback ban (root + metadata) ─────────────────────────── + + +class TestObservabilityCallbackBans: + """The proxy must reject observability credentials, hosts, and project + identifiers regardless of whether they arrive at the request body root, + in ``metadata`` / ``litellm_metadata``, or in a JSON-string-encoded + metadata blob (multipart/``extra_body`` path). + + The ban list is derived from + ``litellm.litellm_core_utils.initialize_dynamic_callback_params._supported_callback_params`` + minus a small ``_SAFE_CLIENT_CALLBACK_PARAMS`` allow-list, plus + ``_EXTRA_BANNED_OBSERVABILITY_PARAMS`` for fields integrations read but + that are not yet in the canonical allow-list. The derivation keeps the + proxy in sync as new integrations are added. + """ + + @pytest.fixture(autouse=True) + def _disable_url_validation(self, monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "user_url_validation", False, raising=False) + + @pytest.mark.parametrize( + "field", + [ + "langfuse_public_key", + "langfuse_secret", + "langfuse_secret_key", + "langsmith_api_key", + "langsmith_project", + "langsmith_tenant_id", + "arize_api_key", + "arize_space_key", + "arize_space_id", + "posthog_api_key", + "posthog_api_url", + "braintrust_api_key", + "braintrust_project", + "phoenix_project_name", + "wandb_api_key", + "weave_project_id", + "gcs_bucket_name", + "gcs_path_service_account", + "humanloop_api_key", + "lunary_public_key", + ], + ) + def test_observability_field_in_request_body_root_is_rejected(self, field): + with pytest.raises(ValueError) as exc: + is_request_body_safe( + request_body={"model": "gpt-4", field: "attacker-value"}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + assert field in str(exc.value) + + @pytest.mark.parametrize( + "metadata_key", + ["metadata", "litellm_metadata"], + ) + @pytest.mark.parametrize( + "field", + [ + "langfuse_host", + "langfuse_secret_key", + "langsmith_api_key", + "posthog_api_url", + "braintrust_project", + "phoenix_project_name", + ], + ) + def test_observability_field_in_metadata_dict_is_rejected( + self, metadata_key, field + ): + # Verifies the metadata walk: a value smuggled inside ``metadata`` + # or ``litellm_metadata`` is just as dangerous as the same field + # at the body root, and must hit the same gate. + with pytest.raises(ValueError) as exc: + is_request_body_safe( + request_body={ + "model": "gpt-4", + metadata_key: {field: "attacker-value"}, + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + assert field in str(exc.value) + + @pytest.mark.parametrize( + "metadata_key", + ["metadata", "litellm_metadata"], + ) + def test_observability_field_in_json_string_metadata_is_rejected( + self, metadata_key + ): + # Multipart/form-data and ``extra_body`` callers send metadata as a + # JSON-encoded string. The bouncer parses it before applying the + # banned-params check so the JSON-string path can't smuggle past + # the ``isinstance(dict)`` guard. + import json + + with pytest.raises(ValueError) as exc: + is_request_body_safe( + request_body={ + "model": "gpt-4", + metadata_key: json.dumps( + {"langfuse_host": "https://attacker.example"} + ), + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + assert "langfuse_host" in str(exc.value) + + def test_admin_opt_in_allows_metadata_credential_passthrough(self): + # The opt-in gate covers the metadata path the same way it covers + # the root path — operators running BYO observability with + # clientside creds flip a single flag and both paths work. + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "metadata": { + "langfuse_host": "https://my-langfuse.example", + "langfuse_public_key": "pk-mine", + "langfuse_secret_key": "sk-mine", + }, + }, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + def test_safe_per_request_observability_metadata_is_allowed(self): + # Informational fields (sampling rate, prompt version) describe + # the request being logged — they don't choose the destination or + # credentials, so they must remain accepted from clients without + # the opt-in flag. + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "metadata": { + "langfuse_prompt_version": "v2", + "langsmith_sampling_rate": 0.1, + }, + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + +def test_observability_ban_covers_canonical_supported_callback_params(): + """Guard test: every entry in the canonical + ``_supported_callback_params`` allow-list must end up either banned by + the proxy or explicitly safe-listed. New integrations added to that + list are banned by default (the safe failure mode); flagging them as + safe is an explicit decision recorded in + ``_SAFE_CLIENT_CALLBACK_PARAMS``.""" + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + _supported_callback_params, + ) + from litellm.proxy.auth.auth_utils import ( + _BANNED_REQUEST_BODY_PARAMS, + _SAFE_CLIENT_CALLBACK_PARAMS, + ) + + banned = set(_BANNED_REQUEST_BODY_PARAMS) + for param in _supported_callback_params: + assert param in banned or param in _SAFE_CLIENT_CALLBACK_PARAMS, ( + f"{param} is in _supported_callback_params but neither banned nor " + f"safe-listed. Add it to _SAFE_CLIENT_CALLBACK_PARAMS if it is an " + f"informational per-request field; otherwise the derivation will " + f"ban it automatically." + ) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 4e24d8af65..d2a1468be2 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -596,6 +596,59 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "pillar_response_headers" not in snapshot_body["metadata"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "control_field", + ["callbacks", "service_callback", "logger_fn", "litellm_disabled_callbacks"], +) +async def test_add_litellm_data_to_request_strips_callback_control_fields( + control_field, +): + """``callbacks`` / ``service_callback`` / ``logger_fn`` get appended to + the worker-wide ``litellm.{input,success,failure,_async_*,service}_callback`` + lists and ``litellm.user_logger_fn`` from inside ``function_setup`` — + one request poisons every subsequent caller in that worker. + ``litellm_disabled_callbacks`` is the inverse: a request-body value + silently disables admin-configured audit/observability for the call. + None has a documented per-request use, so all four are stripped at + the proxy boundary alongside the existing internal-only fields.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + sample_value = ( + ["langfuse"] + if control_field + in ("callbacks", "service_callback", "litellm_disabled_callbacks") + else "module.func" + ) + + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hi"}], + control_field: sample_value, + }, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert control_field not in updated + # The post-strip body snapshot used by audit/spend logging must also + # not retain the attacker-injected control field. + snapshot_body = updated["proxy_server_request"]["body"] + assert control_field not in snapshot_body + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_allows_client_mock_response_with_admin_opt_in(): request_mock = MagicMock(spec=Request) From 01323e890357f34ea39eab025e7fff55391ed26e Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Sun, 3 May 2026 09:18:17 +0000 Subject: [PATCH 09/17] fix(auth): per-param allow must continue, not return early MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pre-existing logic bug in ``_check_banned_params``: when the deployment-level ``configurable_clientside_auth_params`` permitted one banned field, the loop ``return``-ed on the first match instead of ``continue``-ing, so any other banned param later in the same body or metadata dict was never checked. This PR's metadata walk multiplies the surface where that bypass matters — a body pairing an allowed ``api_base`` with an observability credential like ``langfuse_host`` would silently pass. Proxy-wide ``allow_client_side_credentials`` keeps ``return`` (it's a global opt-in for every banned param). The per-param branch becomes ``continue`` so only the one explicitly-permitted field is skipped. Adds a regression test that exercises the api_base + langfuse_host pair. Co-Authored-By: Claude Opus 4.7 (1M context) --- litellm/proxy/auth/auth_utils.py | 9 +++++- .../proxy/auth/test_auth_utils.py | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 84fb2f8a95..9a6fc95f14 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -279,6 +279,8 @@ def _check_banned_params( if param not in body: continue if general_settings.get("allow_client_side_credentials") is True: + # Proxy-wide opt-in: every banned param is permitted, exit + # entirely so the rest of the loop doesn't waste work. return if ( _allow_model_level_clientside_configurable_parameters( @@ -289,7 +291,12 @@ def _check_banned_params( ) is True ): - return + # Per-param opt-in: only THIS param is permitted by the + # deployment's ``configurable_clientside_auth_params``. Skip + # to the next banned param so a body that pairs an allowed + # ``api_base`` with an unallowed ``langfuse_host`` is still + # rejected for the second field. + continue raise ValueError( f"Rejected Request: {param} is not allowed in request body. " "Clientside passthrough requires explicit admin opt-in via " diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 58a55f8aa6..7c04a4f61f 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1455,6 +1455,36 @@ class TestObservabilityCallbackBans: ) +def test_model_level_allow_does_not_skip_subsequent_banned_params(monkeypatch): + """Greptile P1: ``_check_banned_params`` previously ``return``-ed when a + deployment's ``configurable_clientside_auth_params`` permitted one + banned field, exiting before any later banned field in the same body + was checked. The metadata walk this PR adds multiplies the surface + where that bypass matters: a body pairing a model-level-allowed + ``api_base`` with an observability credential like ``langfuse_host`` + must still reject on the second field, not silently pass.""" + from litellm.proxy.auth import auth_utils + + monkeypatch.setattr( + auth_utils, + "_allow_model_level_clientside_configurable_parameters", + lambda model, param, request_body_value, llm_router: param == "api_base", + ) + + with pytest.raises(ValueError) as exc: + is_request_body_safe( + request_body={ + "model": "gpt-4", + "api_base": "https://allowed-by-deployment.example", + "langfuse_host": "https://attacker.example", + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + assert "langfuse_host" in str(exc.value) + + def test_observability_ban_covers_canonical_supported_callback_params(): """Guard test: every entry in the canonical ``_supported_callback_params`` allow-list must end up either banned by From b87c2f66a6287451911538db19ee9dda0ca70c31 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Sun, 3 May 2026 10:08:28 +0000 Subject: [PATCH 10/17] fix(vector_store): resolve embedding config at request time, never persist creds The vector store create/update path previously called ``_resolve_embedding_config`` against the admin-configured router/DB model and persisted the resolved ``litellm_embedding_config`` dict (``api_key`` / ``api_base`` / ``api_version``) into the ``litellm_managedvectorstorestable.litellm_params`` column. Because the resolver expanded ``os.environ/...`` references via ``get_secret``, the DB row carried cleartext provider credentials, and the ``/vector_store/{new,info,update,list}`` responses returned them to any authenticated caller who could supply a known admin model name. Move the auto-resolve out of ``create_vector_store_in_db`` and out of the update path. Persist only the user-supplied ``litellm_embedding_model`` reference. Resolve at request-handling time inside ``_update_request_data_with_litellm_managed_vector_store_registry`` so the resolved config lives in the per-request ``data`` dict and is garbage-collected after the response. Legacy rows that were created by an earlier proxy version and already carry a resolved ``litellm_embedding_config`` skip the re-resolution and pass through unchanged so embedding calls keep working. The ``new_vector_store`` response now also runs the existing ``_redact_sensitive_litellm_params`` masker (already used by ``info``, ``update``, and ``list``), defending against caller-supplied cleartext on the create path and against legacy rows whose persisted credentials are still in the database. Existing tests that asserted the old write-time-resolve behaviour are updated to assert the new persistence shape (no embedding config stored, just the model reference). Two new tests cover the use-time path: one asserting fresh resolution happens when a row carries only the model reference, the other asserting legacy rows with persisted config skip re-resolution and continue to work. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../proxy/vector_store_endpoints/endpoints.py | 22 +++ .../management_endpoints.py | 57 ++++--- .../test_vector_store_endpoints.py | 146 +++++++++++++++--- 3 files changed, 172 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 86e316e7f4..0f9753a303 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -56,6 +56,28 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( if "litellm_params" in vector_store_to_run: litellm_params = vector_store_to_run.get("litellm_params", {}) or {} + # Resolve ``litellm_embedding_config`` here, at request-handling + # time, instead of at row-creation time. The resolved + # ``api_key`` / ``api_base`` / ``api_version`` lives only in + # this per-request ``data`` dict and is never persisted. + # Legacy rows that already carry a resolved (cleartext) + # ``litellm_embedding_config`` skip the lookup and pass through + # unchanged so the embed call keeps working. + embedding_model = litellm_params.get("litellm_embedding_model") + if embedding_model and not litellm_params.get("litellm_embedding_config"): + from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.vector_store_endpoints.management_endpoints import ( + _resolve_embedding_config, + ) + + resolved_config = await _resolve_embedding_config( + embedding_model=embedding_model, prisma_client=prisma_client + ) + if resolved_config: + litellm_params = { + **litellm_params, + "litellm_embedding_config": resolved_config, + } data.update(litellm_params) return data diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 99a2085bfc..bff43cacaa 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -432,20 +432,17 @@ async def create_vector_store_in_db( if user_id is not None: data_to_create["user_id"] = user_id - # Handle litellm_params - always provide at least an empty dict + # Handle litellm_params - always provide at least an empty dict. + # The earlier behaviour resolved ``litellm_embedding_config`` from the + # admin-configured router/DB model and persisted the cleartext result + # (``api_key``, ``api_base``, ``api_version``) into this row. That + # exposed every env-stored embedding-model credential on the + # ``/vector_store/{new,info,update,list}`` responses. Keep the user's + # raw ``litellm_embedding_model`` reference; resolution now happens in + # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # at request-handling time so the cleartext config exists only in + # per-request memory and never reaches the database. if litellm_params: - # Auto-resolve embedding config if embedding model is provided but config is not - embedding_model = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): - resolved_config = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - litellm_params["litellm_embedding_config"] = resolved_config - verbose_proxy_logger.info( - f"Auto-resolved embedding config for model {embedding_model}" - ) - litellm_params_dict = GenericLiteLLMParams(**litellm_params).model_dump( exclude_none=True ) @@ -531,10 +528,19 @@ async def new_vector_store( user_id=user_api_key_dict.user_id, ) + # Apply the same litellm_params redaction the list / info / update + # endpoints already use, so a caller-supplied credential or a + # cleartext value persisted by an earlier proxy version doesn't + # come back in the response. + response_vs = LiteLLM_ManagedVectorStore(**new_vector_store) + response_vs["litellm_params"] = _redact_sensitive_litellm_params( + new_vector_store.get("litellm_params") + ) + return { "status": "success", "message": f"Vector store {vector_store.get('vector_store_id')} created successfully", - "vector_store": new_vector_store, + "vector_store": response_vs, } except Exception as e: verbose_proxy_logger.exception(f"Error creating vector store: {str(e)}") @@ -865,24 +871,15 @@ async def update_vector_store( update_data["vector_store_metadata"] ) - # Handle litellm_params if provided + # Handle litellm_params if provided. As with the create path, the + # embedding-config auto-resolve previously persisted cleartext + # credentials into the row; resolution now happens at request- + # handling time in + # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # so this row only ever stores the user-supplied + # ``litellm_embedding_model`` reference. if "litellm_params" in update_data: _input_litellm_params: dict = update_data.get("litellm_params", {}) or {} - - # Auto-resolve embedding config if embedding model is provided but config is not - embedding_model = _input_litellm_params.get("litellm_embedding_model") - if embedding_model and not _input_litellm_params.get( - "litellm_embedding_config" - ): - resolved_config = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - _input_litellm_params["litellm_embedding_config"] = resolved_config - verbose_proxy_logger.info( - f"Auto-resolved embedding config for model {embedding_model}" - ) - litellm_params_dict = GenericLiteLLMParams( **_input_litellm_params ).model_dump(exclude_none=True) diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index e67a04c749..7f5557f41e 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -170,6 +170,95 @@ async def test_update_request_data_with_litellm_managed_vector_store_registry(): assert result == original_data +@pytest.mark.asyncio +async def test_update_request_data_resolves_embedding_config_at_use_time(): + """When the persisted vector store row carries only a + ``litellm_embedding_model`` reference (the new behaviour after + moving the auto-resolve out of write time), the request-handling + layer must resolve the embedding config so the downstream embed + call still has ``api_key`` / ``api_base`` / ``api_version``. The + resolved config lives in this per-request data dict only — never + persisted.""" + mock_vector_store: LiteLLM_ManagedVectorStore = { + "vector_store_id": "test_store", + "custom_llm_provider": "azure_ai", + "litellm_params": { + "litellm_embedding_model": "azure/text-embedding-3-large", + # Note: no litellm_embedding_config persisted + }, + } + + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( + mock_vector_store + ) + + resolved = { + "api_key": "use-time-resolved-key", + "api_base": "https://my-azure.example", + "api_version": "2024-09-01", + } + + with ( + patch.object(litellm, "vector_store_registry", mock_registry), + patch( + "litellm.proxy.vector_store_endpoints.management_endpoints._resolve_embedding_config", + new=AsyncMock(return_value=resolved), + ), + ): + result = await _update_request_data_with_litellm_managed_vector_store_registry( + data={}, vector_store_id="test_store" + ) + + assert result["litellm_embedding_model"] == "azure/text-embedding-3-large" + assert result["litellm_embedding_config"] == resolved + + +@pytest.mark.asyncio +async def test_update_request_data_passes_through_legacy_embedding_config(): + """A vector store row created by an older proxy version may already + carry a fully-resolved ``litellm_embedding_config`` in its persisted + ``litellm_params`` (the very leak this PR closes). Those legacy rows + must still work — the use-time resolver skips re-resolution when + the config is already present so the embed call keeps succeeding.""" + legacy_config = { + "api_key": "legacy-cleartext-key", + "api_base": "https://legacy-azure.example", + "api_version": "2024-01-01", + } + mock_vector_store: LiteLLM_ManagedVectorStore = { + "vector_store_id": "legacy_store", + "custom_llm_provider": "azure_ai", + "litellm_params": { + "litellm_embedding_model": "azure/text-embedding-3-large", + "litellm_embedding_config": legacy_config, + }, + } + + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( + mock_vector_store + ) + + resolve_mock = AsyncMock( + return_value={"api_key": "should-not-be-used", "api_base": "wrong"} + ) + + with ( + patch.object(litellm, "vector_store_registry", mock_registry), + patch( + "litellm.proxy.vector_store_endpoints.management_endpoints._resolve_embedding_config", + new=resolve_mock, + ), + ): + result = await _update_request_data_with_litellm_managed_vector_store_registry( + data={}, vector_store_id="legacy_store" + ) + + assert result["litellm_embedding_config"] == legacy_config + resolve_mock.assert_not_awaited() + + class TestCheckVectorStorePermission: """Test suite for check_vector_store_permission function.""" @@ -1417,20 +1506,31 @@ async def test_new_vector_store_auto_resolves_embedding_config(): ) assert result["status"] == "success" - # Verify that embedding config was resolved and included in the create call + # Auto-resolve no longer happens at create time — the persisted row + # carries only the model reference, never the resolved cleartext + # credential. Resolution now happens at request-handling time inside + # ``_update_request_data_with_litellm_managed_vector_store_registry``, + # where the resolved config lives in per-request memory and is never + # written to the database. litellm_params_json = captured_create_data.get("litellm_params") assert litellm_params_json is not None litellm_params_dict = json.loads(litellm_params_json) - assert "litellm_embedding_config" in litellm_params_dict - assert ( - litellm_params_dict["litellm_embedding_config"]["api_key"] == "resolved-api-key" - ) - assert ( - litellm_params_dict["litellm_embedding_config"]["api_base"] - == "https://api.openai.com" - ) - assert ( - litellm_params_dict["litellm_embedding_config"]["api_version"] == "2024-01-01" + assert "litellm_embedding_config" not in litellm_params_dict + assert litellm_params_dict["litellm_embedding_model"] == "text-embedding-ada-002" + + # The response must also not echo a cleartext credential — even on + # the create response, where redaction guards against caller-supplied + # cleartext or pre-existing rows that were created by an earlier + # proxy version. + response_vs = result["vector_store"] + response_params = response_vs.get("litellm_params") + # The redact helper preserves the persisted shape (string or dict); + # serialise to text either way and assert the cleartext credential + # never appears. + assert "resolved-api-key" not in ( + response_params + if isinstance(response_params, str) + else json.dumps(response_params or {}) ) @@ -1687,21 +1787,21 @@ async def test_new_vector_store_auto_resolves_from_router(): ) assert result["status"] == "success" - # Verify that embedding config was resolved from router and included in the create call + # Resolution against the router happens at request-handling time now, + # not at row creation. The persisted ``litellm_params`` carries only + # the model reference, never the cleartext credential. litellm_params_json = captured_create_data.get("litellm_params") assert litellm_params_json is not None litellm_params_dict = json.loads(litellm_params_json) - assert "litellm_embedding_config" in litellm_params_dict - assert ( - litellm_params_dict["litellm_embedding_config"]["api_key"] - == "router-resolved-api-key" - ) - assert ( - litellm_params_dict["litellm_embedding_config"]["api_base"] - == "https://router-resolved-base.com" - ) - assert ( - litellm_params_dict["litellm_embedding_config"]["api_version"] == "2024-03-01" + assert "litellm_embedding_config" not in litellm_params_dict + assert litellm_params_dict["litellm_embedding_model"] == "config-embedding-model" + + response_vs = result["vector_store"] + response_params = response_vs.get("litellm_params") + assert "router-resolved-api-key" not in ( + response_params + if isinstance(response_params, str) + else json.dumps(response_params or {}) ) From 38511675b16f1d2bfe71218e5989ac21a354c24d Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Sun, 3 May 2026 10:17:50 +0000 Subject: [PATCH 11/17] fix(vector_store): tighten registry-mutation comment and dedupe test helpers --- .../proxy/vector_store_endpoints/endpoints.py | 11 ++++-- .../test_vector_store_endpoints.py | 39 ++++++++++--------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 0f9753a303..ccf15c206b 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -8,6 +8,9 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import jsonify_object +from litellm.proxy.vector_store_endpoints.management_endpoints import ( + _resolve_embedding_config, +) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store, get_litellm_managed_vector_store, @@ -66,14 +69,16 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( embedding_model = litellm_params.get("litellm_embedding_model") if embedding_model and not litellm_params.get("litellm_embedding_config"): from litellm.proxy.proxy_server import prisma_client - from litellm.proxy.vector_store_endpoints.management_endpoints import ( - _resolve_embedding_config, - ) resolved_config = await _resolve_embedding_config( embedding_model=embedding_model, prisma_client=prisma_client ) if resolved_config: + # Build a fresh dict via spread instead of mutating + # ``litellm_params`` in place — the registry hands back + # a reference to its cached object, so an in-place + # update would persist the resolved cleartext into the + # in-memory cache for the lifetime of the process. litellm_params = { **litellm_params, "litellm_embedding_config": resolved_config, diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 7f5557f41e..8fc7bc4e26 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -36,6 +36,20 @@ from litellm.proxy.vector_store_endpoints.utils import ( from litellm.types.utils import LlmProviders +def _serialize_litellm_params(litellm_params): + """Serialize ``litellm_params`` to a string for substring assertions. + + The redact helper preserves the persisted shape — string in, string + out; dict in, dict out — so callers that just want to assert "this + secret never appears" need a single text representation either way. + """ + import json + + if isinstance(litellm_params, str): + return litellm_params + return json.dumps(litellm_params or {}) + + @pytest.mark.asyncio async def test_router_avector_store_search_passes_correct_args(): """ @@ -202,7 +216,7 @@ async def test_update_request_data_resolves_embedding_config_at_use_time(): with ( patch.object(litellm, "vector_store_registry", mock_registry), patch( - "litellm.proxy.vector_store_endpoints.management_endpoints._resolve_embedding_config", + "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", new=AsyncMock(return_value=resolved), ), ): @@ -240,14 +254,12 @@ async def test_update_request_data_passes_through_legacy_embedding_config(): mock_vector_store ) - resolve_mock = AsyncMock( - return_value={"api_key": "should-not-be-used", "api_base": "wrong"} - ) + resolve_mock = AsyncMock() with ( patch.object(litellm, "vector_store_registry", mock_registry), patch( - "litellm.proxy.vector_store_endpoints.management_endpoints._resolve_embedding_config", + "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", new=resolve_mock, ), ): @@ -1523,14 +1535,8 @@ async def test_new_vector_store_auto_resolves_embedding_config(): # cleartext or pre-existing rows that were created by an earlier # proxy version. response_vs = result["vector_store"] - response_params = response_vs.get("litellm_params") - # The redact helper preserves the persisted shape (string or dict); - # serialise to text either way and assert the cleartext credential - # never appears. - assert "resolved-api-key" not in ( - response_params - if isinstance(response_params, str) - else json.dumps(response_params or {}) + assert "resolved-api-key" not in _serialize_litellm_params( + response_vs.get("litellm_params") ) @@ -1797,11 +1803,8 @@ async def test_new_vector_store_auto_resolves_from_router(): assert litellm_params_dict["litellm_embedding_model"] == "config-embedding-model" response_vs = result["vector_store"] - response_params = response_vs.get("litellm_params") - assert "router-resolved-api-key" not in ( - response_params - if isinstance(response_params, str) - else json.dumps(response_params or {}) + assert "router-resolved-api-key" not in _serialize_litellm_params( + response_vs.get("litellm_params") ) From 74b4eab364348aa8cb22e4e15060ba78e083bd42 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Sun, 3 May 2026 10:27:53 +0000 Subject: [PATCH 12/17] fix(vector_store): cache use-time embedding-config resolution Hold the resolved config in a process-memory TTL cache so the request-handling path doesn't run litellm_proxymodeltable.find_first on every vector-store call. --- .../management_endpoints.py | 35 +++++++++++++ .../test_vector_store_endpoints.py | 50 +++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index bff43cacaa..cbb3d92718 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker @@ -45,6 +46,28 @@ _LITELLM_PARAMS_MASKER = SensitiveDataMasker() _REDACT_LITELLM_PARAMS_MAX_DEPTH = 10 +# Use-time embedding-config resolution runs on every vector-store request +# whose persisted row carries only a model reference (the post-fix shape). +# Without a cache, that's one ``litellm_proxymodeltable.find_first`` per +# request — the no-DB-in-critical-path rule. Hold the resolved config in +# memory for a short TTL so a hot model name pays the DB lookup at most +# once per ``_EMBEDDING_CONFIG_CACHE_TTL`` seconds. Cleartext credentials +# only ever live in process memory (never persisted, never echoed in +# management responses), so the cache doesn't widen the disclosure surface. +_EMBEDDING_CONFIG_CACHE_TTL = 60 +_EMBEDDING_CONFIG_CACHE_MAX_SIZE = 256 +_embedding_config_cache: Optional[InMemoryCache] = None + + +def _get_embedding_config_cache() -> InMemoryCache: + global _embedding_config_cache + if _embedding_config_cache is None: + _embedding_config_cache = InMemoryCache( + max_size_in_memory=_EMBEDDING_CONFIG_CACHE_MAX_SIZE, + default_ttl=_EMBEDDING_CONFIG_CACHE_TTL, + ) + return _embedding_config_cache + def _redact_sensitive_litellm_params(litellm_params: Any, _depth: int = 0) -> Any: """ @@ -303,6 +326,11 @@ async def _resolve_embedding_config( This function first checks the router for config-defined models, then falls back to the database. This allows users to use models defined in either location. + Results are cached in process memory for ``_EMBEDDING_CONFIG_CACHE_TTL`` + seconds so the request-handling path doesn't hit the database on every + vector-store call. Negative results (model not found) are intentionally + not cached to avoid blocking a freshly-added model behind the TTL. + Args: embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") prisma_client: The Prisma client instance @@ -314,6 +342,11 @@ async def _resolve_embedding_config( if not embedding_model: return None + cache = _get_embedding_config_cache() + cached = cache.get_cache(embedding_model) + if cached is not None: + return cached + # Import llm_router if not provided if llm_router is None: try: @@ -330,6 +363,7 @@ async def _resolve_embedding_config( verbose_proxy_logger.debug( f"Resolved embedding config from router for model {embedding_model}" ) + cache.set_cache(embedding_model, router_config) return router_config # Fall back to database @@ -341,6 +375,7 @@ async def _resolve_embedding_config( verbose_proxy_logger.debug( f"Resolved embedding config from database for model {embedding_model}" ) + cache.set_cache(embedding_model, db_config) return db_config verbose_proxy_logger.debug( diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 8fc7bc4e26..81b67e8bc5 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -50,6 +50,19 @@ def _serialize_litellm_params(litellm_params): return json.dumps(litellm_params or {}) +@pytest.fixture(autouse=True) +def _reset_embedding_config_cache(): + """The use-time embedding-config resolver caches results in process + memory across calls. Reset it before every test so the resolver + actually exercises the router/DB path under test instead of returning + a value cached by an earlier test.""" + from litellm.proxy.vector_store_endpoints import management_endpoints + + management_endpoints._embedding_config_cache = None + yield + management_endpoints._embedding_config_cache = None + + @pytest.mark.asyncio async def test_router_avector_store_search_passes_correct_args(): """ @@ -1684,6 +1697,43 @@ async def test_resolve_embedding_config_tries_router_then_db(): mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_not_called() +@pytest.mark.asyncio +async def test_resolve_embedding_config_caches_result(): + """The first lookup should hit the router/DB; subsequent lookups for + the same model name should return the cached value without touching + the router or the database.""" + from litellm.types.router import Deployment, LiteLLM_Params + + mock_prisma_client = MagicMock() + mock_router = MagicMock() + + mock_litellm_params = MagicMock(spec=LiteLLM_Params) + mock_litellm_params.api_key = "router-api-key" + mock_litellm_params.api_base = "https://router-api-base.com" + mock_litellm_params.api_version = None + + mock_deployment = MagicMock(spec=Deployment) + mock_deployment.litellm_params = mock_litellm_params + mock_router.get_deployment_by_model_group_name.return_value = mock_deployment + + first = await _resolve_embedding_config( + embedding_model="cached-model", + prisma_client=mock_prisma_client, + llm_router=mock_router, + ) + assert first is not None + assert mock_router.get_deployment_by_model_group_name.call_count == 1 + + second = await _resolve_embedding_config( + embedding_model="cached-model", + prisma_client=mock_prisma_client, + llm_router=mock_router, + ) + assert second == first + # Router (and by extension the DB) was not consulted again. + assert mock_router.get_deployment_by_model_group_name.call_count == 1 + + @pytest.mark.asyncio async def test_resolve_embedding_config_falls_back_to_db(): """Test that _resolve_embedding_config falls back to DB when router doesn't have the model.""" From a05d3b585170a5db361ece0c1f227c134686ac3d Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 11:18:41 -0700 Subject: [PATCH 13/17] Fix qdrant semantic cache miss metadata --- litellm/caching/qdrant_semantic_cache.py | 6 ++ .../caching/test_qdrant_semantic_cache.py | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index e7436e2ca6..47359a6023 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -340,9 +340,11 @@ class QdrantSemanticCache(BaseCache): results = search_response.json()["result"] if results is None: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None if isinstance(results, list): if len(results) == 0: + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None similarity = results[0]["score"] @@ -358,6 +360,10 @@ class QdrantSemanticCache(BaseCache): print_verbose( f"semantic cache: similarity threshold: {self.similarity_threshold}, similarity: {similarity}, prompt: {prompt}, closest_cached_prompt: {cached_prompt}" ) + + # update kwargs["metadata"] with similarity, don't rewrite the original metadata + kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity + if similarity >= self.similarity_threshold: # cache hit ! cached_value = payload["response"] diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 71dbd24063..949e6ccc29 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -241,6 +241,76 @@ def test_qdrant_semantic_cache_payload_index_exception_is_non_blocking(): qdrant_cache.sync_client.put.assert_called_once() +def _mock_qdrant_get_cache_result(qdrant_result): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.embedding_model = "text-embedding-ada-002" + qdrant_cache.qdrant_api_base = "http://test.qdrant.local" + qdrant_cache.collection_name = "test_collection" + qdrant_cache.headers = { + "Content-Type": "application/json", + "api-key": "test_key", + } + qdrant_cache.similarity_threshold = 0.8 + qdrant_cache.sync_client = MagicMock() + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = {"result": qdrant_result} + qdrant_cache.sync_client.post.return_value = mock_search_response + + return qdrant_cache, QdrantSemanticCache + + +@pytest.mark.parametrize("qdrant_result", [None, []]) +def test_qdrant_semantic_cache_get_cache_sets_metadata_on_empty_miss(qdrant_result): + qdrant_cache, _ = _mock_qdrant_get_cache_result(qdrant_result) + metadata = {} + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of Spain?"}], + metadata=metadata, + ) + + assert result is None + assert metadata["semantic-similarity"] == 0.0 + + +def test_qdrant_semantic_cache_get_cache_sets_metadata_on_below_threshold_miss(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache, _ = _mock_qdrant_get_cache_result( + [ + { + "payload": { + QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "test_key", + "text": "What is the capital of Spain?", + "response": '{"id": "test-456"}', + }, + "score": 0.7, + } + ] + ) + metadata = {} + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of Spain?"}], + metadata=metadata, + ) + + assert result is None + assert metadata["semantic-similarity"] == 0.7 + + def test_qdrant_semantic_cache_get_cache_miss(): """ Test QDRANT semantic cache get method when there's a cache miss. From 9f1feaadeb97161cf2d1d9c750e0f523c3b6a202 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 14:23:31 -0700 Subject: [PATCH 14/17] Clean up Redis semantic cache isolation fallback --- litellm/caching/redis_semantic_cache.py | 30 ++++++++----------- .../caching/test_redis_semantic_cache.py | 2 +- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 27538530bc..4d685c6a87 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -129,7 +129,11 @@ class RedisSemanticCache(BaseCache): cache_vectorizer: Any, ) -> Any: def _is_schema_mismatch(exc: ValueError) -> bool: - return "schema does not match" in str(exc) + error_message = str(exc).lower() + return any( + phrase in error_message + for phrase in ("schema does not match", "index schema") + ) try: return semantic_cache_cls( @@ -284,9 +288,7 @@ class RedisSemanticCache(BaseCache): ttl = self._get_ttl(**kwargs) if ttl is not None: store_kwargs["ttl"] = int(ttl) - self.llmcache.store(prompt, value_str, **store_kwargs) - else: - self.llmcache.store(prompt, value_str, **store_kwargs) + self.llmcache.store(prompt, value_str, **store_kwargs) except Exception as e: print_verbose( f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}" @@ -439,17 +441,11 @@ class RedisSemanticCache(BaseCache): ttl = self._get_ttl(**kwargs) if ttl is not None: store_kwargs["ttl"] = ttl - await self.llmcache.astore( - prompt, - value_str, - **store_kwargs, - ) - else: - await self.llmcache.astore( - prompt, - value_str, - **store_kwargs, - ) + await self.llmcache.astore( + prompt, + value_str, + **store_kwargs, + ) except Exception as e: print_verbose(f"Error in async_set_cache: {str(e)}") @@ -489,9 +485,7 @@ class RedisSemanticCache(BaseCache): # handle results / cache hit if not results: - kwargs.setdefault("metadata", {})[ - "semantic-similarity" - ] = 0.0 # TODO why here but not above?? + kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 return None cache_hit = results[0] diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 02b06bed62..495c72a380 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -195,7 +195,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( side_effect=[ - ValueError("Existing index schema does not match"), + ValueError("stored index schema differs from requested fields"), fallback_cache_mock, ] ) From af7794272b92354f267e6c05c0f0c64660356037 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 14:42:53 -0700 Subject: [PATCH 15/17] Add semantic cache legacy migration flag --- litellm/caching/qdrant_semantic_cache.py | 42 ++++++++-- litellm/caching/redis_semantic_cache.py | 82 +++++++++++++++---- .../caching/test_qdrant_semantic_cache.py | 70 ++++++++++++++++ .../caching/test_redis_semantic_cache.py | 67 ++++++++++++++- 4 files changed, 235 insertions(+), 26 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 47359a6023..a9f01a4c90 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -11,7 +11,8 @@ Has 4 methods: import ast import asyncio import json -from typing import Any, cast +import os +from typing import Any, Optional, cast import litellm from litellm._logging import print_verbose @@ -26,6 +27,9 @@ from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): CACHE_KEY_FIELD_NAME = "litellm_cache_key" + ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR = ( + "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" + ) def __init__( # noqa: PLR0915 self, @@ -37,9 +41,8 @@ class QdrantSemanticCache(BaseCache): embedding_model="text-embedding-ada-002", host_type=None, vector_size=None, + allow_legacy_unscoped_cache_hits: Optional[bool] = None, ): - import os - from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -59,6 +62,16 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model + self.allow_legacy_unscoped_cache_hits = ( + self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) + ) + if self.allow_legacy_unscoped_cache_hits: + print_verbose( + "Qdrant semantic-cache legacy unscoped hits are enabled via " + f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; searches may return " + "pre-isolation cache entries without cache-key payloads. Disable " + "this after warming the key-scoped semantic cache." + ) self.vector_size = ( vector_size if vector_size is not None else QDRANT_VECTOR_SIZE ) @@ -166,6 +179,14 @@ class QdrantSemanticCache(BaseCache): else: raise Exception("Error while creating new collection") + @classmethod + def _get_allow_legacy_unscoped_cache_hits( + cls, allow_legacy_unscoped_cache_hits: Optional[bool] + ) -> bool: + if allow_legacy_unscoped_cache_hits is not None: + return allow_legacy_unscoped_cache_hits + return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" + def _get_cache_logic(self, cached_response: Any): if cached_response is None: return cached_response @@ -187,6 +208,11 @@ class QdrantSemanticCache(BaseCache): ] } + def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None: + if getattr(self, "allow_legacy_unscoped_cache_hits", False): + return + data["filter"] = self._get_qdrant_cache_key_filter(key) + def _ensure_cache_key_payload_index(self) -> None: try: response = self.sync_client.put( @@ -211,8 +237,12 @@ class QdrantSemanticCache(BaseCache): def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: # Legacy Qdrant semantic-cache points stored only prompt text and # response. They cannot be reassigned to the generated LiteLLM cache key - # without risking cross-scope hits, so they must be treated as misses. + # without risking cross-scope hits, so secure mode treats them as misses. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) + if cached_key is None and getattr( + self, "allow_legacy_unscoped_cache_hits", False + ): + return True return cached_key is not None and str(cached_key) == str(key) async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: @@ -329,8 +359,8 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, - "filter": self._get_qdrant_cache_key_filter(key), } + self._add_cache_key_filter_to_search_data(data=data, key=key) search_response = self.sync_client.post( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", @@ -436,8 +466,8 @@ class QdrantSemanticCache(BaseCache): }, "limit": 1, "with_payload": True, - "filter": self._get_qdrant_cache_key_filter(key), } + self._add_cache_key_filter_to_search_data(data=data, key=key) search_response = await self.async_client.post( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 4d685c6a87..35fed977cc 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -36,10 +36,9 @@ class RedisSemanticCache(BaseCache): DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" - CACHE_KEY_FILTERABLE_FIELD: Dict[str, str] = { - "name": CACHE_KEY_FIELD_NAME, - "type": "tag", - } + ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR: str = ( + "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" + ) def __init__( self, @@ -50,6 +49,7 @@ class RedisSemanticCache(BaseCache): similarity_threshold: Optional[float] = None, embedding_model: str = "text-embedding-ada-002", index_name: Optional[str] = None, + allow_legacy_unscoped_cache_hits: Optional[bool] = None, **kwargs, ): """ @@ -91,6 +91,10 @@ class RedisSemanticCache(BaseCache): # While similarity: 1 = most similar, 0 = least similar self.distance_threshold = 1 - similarity_threshold self.embedding_model = embedding_model + self.allow_legacy_unscoped_cache_hits = ( + self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) + ) + self._using_legacy_unscoped_index = False # Set up Redis connection if redis_url is None: @@ -121,6 +125,21 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) + @classmethod + def _get_allow_legacy_unscoped_cache_hits( + cls, allow_legacy_unscoped_cache_hits: Optional[bool] + ) -> bool: + if allow_legacy_unscoped_cache_hits is not None: + return allow_legacy_unscoped_cache_hits + return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" + + @classmethod + def _cache_key_filterable_field(cls) -> Dict[str, str]: + return { + "name": cls.CACHE_KEY_FIELD_NAME, + "type": "tag", + } + def _init_semantic_cache( self, semantic_cache_cls: Any, @@ -141,13 +160,29 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=False, ) except ValueError as exc: if not _is_schema_mismatch(exc): raise + if self.allow_legacy_unscoped_cache_hits: + self._using_legacy_unscoped_index = True + print_verbose( + "Redis semantic-cache legacy unscoped hits are enabled via " + f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; reusing existing " + "index without cache-key isolation. Disable this after warming " + "the isolated semantic cache." + ) + return semantic_cache_cls( + name=index_name, + redis_url=redis_url, + vectorizer=cache_vectorizer, + distance_threshold=self.distance_threshold, + overwrite=False, + ) + isolated_index_name = f"{index_name}_isolated" print_verbose( "Redis semantic-cache existing index schema is not isolated; " @@ -159,7 +194,7 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=False, ) except ValueError as isolated_exc: @@ -175,7 +210,7 @@ class RedisSemanticCache(BaseCache): redis_url=redis_url, vectorizer=cache_vectorizer, distance_threshold=self.distance_threshold, - filterable_fields=[self.CACHE_KEY_FILTERABLE_FIELD], + filterable_fields=[self._cache_key_filterable_field()], overwrite=True, ) @@ -191,6 +226,8 @@ class RedisSemanticCache(BaseCache): cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) if isinstance(cached_key, bytes): cached_key = cached_key.decode("utf-8") + if cached_key is None and getattr(self, "_using_legacy_unscoped_index", False): + return True return cached_key is not None and str(cached_key) == str(key) def _get_ttl(self, **kwargs) -> Optional[int]: @@ -282,7 +319,9 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) - store_kwargs: Dict[str, Any] = {"filters": self._get_cache_filters(key)} + store_kwargs: Dict[str, Any] = {} + if not getattr(self, "_using_legacy_unscoped_index", False): + store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -318,10 +357,12 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - results = self.llmcache.check( - prompt=prompt, - filter_expression=self._get_cache_key_filter_expression(key), - ) + check_kwargs: Dict[str, Any] = {"prompt": prompt} + if not getattr(self, "_using_legacy_unscoped_index", False): + check_kwargs["filter_expression"] = ( + self._get_cache_key_filter_expression(key) + ) + results = self.llmcache.check(**check_kwargs) # Return None if no similar prompts found if not results: @@ -434,8 +475,9 @@ class RedisSemanticCache(BaseCache): store_kwargs: Dict[str, Any] = { "vector": prompt_embedding, - "filters": self._get_cache_filters(key), } + if not getattr(self, "_using_legacy_unscoped_index", False): + store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -477,11 +519,15 @@ class RedisSemanticCache(BaseCache): # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - results = await self.llmcache.acheck( - prompt=prompt, - vector=prompt_embedding, - filter_expression=self._get_cache_key_filter_expression(key), - ) + check_kwargs: Dict[str, Any] = { + "prompt": prompt, + "vector": prompt_embedding, + } + if not getattr(self, "_using_legacy_unscoped_index", False): + check_kwargs["filter_expression"] = ( + self._get_cache_key_filter_expression(key) + ) + results = await self.llmcache.acheck(**check_kwargs) # handle results / cache hit if not results: diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 949e6ccc29..9b987a6d4f 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -208,6 +208,76 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert metadata["semantic-similarity"] == 0.0 +def test_qdrant_semantic_cache_allows_legacy_unscoped_hit_with_flag(monkeypatch): + monkeypatch.setenv("LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS", "true") + + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), + ): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = { + "result": [ + { + "payload": { + "text": "What is the capital of France?", + "response": '{"id": "test-123"}', + }, + "score": 0.9, + } + ] + } + qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + metadata = {} + result = qdrant_cache.get_cache( + key="test_key", + messages=[{"content": "What is the capital of France?"}], + metadata=metadata, + ) + + assert result == {"id": "test-123"} + assert metadata["semantic-similarity"] == 0.9 + assert "filter" not in qdrant_cache.sync_client.post.call_args.kwargs["json"] + + +def test_qdrant_semantic_cache_legacy_mode_rejects_wrong_key_hit(): + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) + qdrant_cache.allow_legacy_unscoped_cache_hits = True + + assert qdrant_cache._payload_matches_cache_key(payload={}, key="test_key") + assert not qdrant_cache._payload_matches_cache_key( + payload={QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "other_key"}, + key="test_key", + ) + + def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 495c72a380..bebe1f757b 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -228,10 +228,50 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): == "existing_index_isolated" ) assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ - RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + RedisSemanticCache._cache_key_filterable_field() ] +def test_redis_semantic_cache_can_reuse_legacy_unscoped_index(monkeypatch): + fallback_cache_mock = MagicMock() + semantic_cache_mock = MagicMock( + side_effect=[ + ValueError("Existing index schema does not match"), + fallback_cache_mock, + ] + ) + custom_vectorizer_mock = MagicMock() + + with patch.dict( + "sys.modules", + { + "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), + "redisvl.utils.vectorize": MagicMock( + CustomTextVectorizer=custom_vectorizer_mock + ), + }, + ): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.setenv("REDIS_PASSWORD", "test_password") + monkeypatch.setenv( + RedisSemanticCache.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "true" + ) + + redis_semantic_cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + + assert redis_semantic_cache.llmcache is fallback_cache_mock + assert redis_semantic_cache._using_legacy_unscoped_index is True + assert semantic_cache_mock.call_count == 2 + assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index" + assert "filterable_fields" not in semantic_cache_mock.call_args_list[1].kwargs + + def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( @@ -270,7 +310,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ - RedisSemanticCache.CACHE_KEY_FILTERABLE_FIELD + RedisSemanticCache._cache_key_filterable_field() ] @@ -332,6 +372,29 @@ def test_redis_semantic_cache_matches_bytes_cache_key(): ) +def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache._using_legacy_unscoped_index = False + + cache_hit = { + "prompt": "What is the capital of France?", + "response": '{"content": "Paris"}', + "vector_distance": 0.1, + } + assert not redis_semantic_cache._cache_hit_matches_key( + cache_hit=cache_hit, + key="test_key", + ) + + redis_semantic_cache._using_legacy_unscoped_index = True + assert redis_semantic_cache._cache_hit_matches_key( + cache_hit=cache_hit, + key="test_key", + ) + + def test_redis_semantic_cache_builds_filter_expression(monkeypatch): class FakeTag: def __init__(self, field_name): From 7d7244986ed242e6807142ee2559ddf64d7b5c6f Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 22:10:59 +0000 Subject: [PATCH 16/17] chore(caching): annotate qdrant quantization_params dict type Mypy infers the dict's value type from the first branch (Dict[str, bool]) which clashes with the scalar branch's mixed-type inner dict. Explicit Dict[str, Any] annotation lifts the inference. --- litellm/caching/qdrant_semantic_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index a9f01a4c90..9206f326ec 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -12,7 +12,7 @@ import ast import asyncio import json import os -from typing import Any, Optional, cast +from typing import Any, Dict, Optional, cast import litellm from litellm._logging import print_verbose @@ -135,6 +135,7 @@ class QdrantSemanticCache(BaseCache): ) self._ensure_cache_key_payload_index() else: + quantization_params: Dict[str, Any] if quantization_config is None or quantization_config == "binary": quantization_params = { "binary": { From a2473ef0c23275bbfcee6a1a35cda684cf522492 Mon Sep 17 00:00:00 2001 From: user <70670632+stuxf@users.noreply.github.com> Date: Mon, 4 May 2026 22:16:30 +0000 Subject: [PATCH 17/17] chore(caching): remove allow_legacy_unscoped_cache_hits opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag was an opt-in escape hatch for the cross-tenant leak the rest of the patch closes — flipping it on (env var or constructor param) re-enables exactly the VERIA-54 primitive on either backend. There is no operational need that the secure path doesn't already meet: - Qdrant: legacy points without ``litellm_cache_key`` payload are excluded by the must-clause filter and treated as misses; new sets populate the cache key, so cold-start lasts only as long as the natural cache rebuild. - Redis: existing unscoped index can't carry the new schema; the init path falls back to ``{name}_isolated`` (and recreates it on stale schema), leaving the legacy index untouched. Drop the constructor param, env-var fallback, ``_using_legacy_unscoped_index`` flag, the legacy-reuse branch in ``_init_semantic_cache``, and the matching guards in set/get paths. Update tests to drop the legacy-mode cases and assert the secure-only behaviour. --- litellm/caching/qdrant_semantic_cache.py | 37 ++-------- litellm/caching/redis_semantic_cache.py | 59 +++------------- .../caching/test_qdrant_semantic_cache.py | 70 ------------------- .../caching/test_redis_semantic_cache.py | 51 +------------- 4 files changed, 19 insertions(+), 198 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 9206f326ec..cb521efca0 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -12,7 +12,7 @@ import ast import asyncio import json import os -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, cast import litellm from litellm._logging import print_verbose @@ -27,9 +27,6 @@ from .base_cache import BaseCache class QdrantSemanticCache(BaseCache): CACHE_KEY_FIELD_NAME = "litellm_cache_key" - ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR = ( - "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" - ) def __init__( # noqa: PLR0915 self, @@ -41,7 +38,6 @@ class QdrantSemanticCache(BaseCache): embedding_model="text-embedding-ada-002", host_type=None, vector_size=None, - allow_legacy_unscoped_cache_hits: Optional[bool] = None, ): from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -62,16 +58,6 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model - self.allow_legacy_unscoped_cache_hits = ( - self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) - ) - if self.allow_legacy_unscoped_cache_hits: - print_verbose( - "Qdrant semantic-cache legacy unscoped hits are enabled via " - f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; searches may return " - "pre-isolation cache entries without cache-key payloads. Disable " - "this after warming the key-scoped semantic cache." - ) self.vector_size = ( vector_size if vector_size is not None else QDRANT_VECTOR_SIZE ) @@ -180,14 +166,6 @@ class QdrantSemanticCache(BaseCache): else: raise Exception("Error while creating new collection") - @classmethod - def _get_allow_legacy_unscoped_cache_hits( - cls, allow_legacy_unscoped_cache_hits: Optional[bool] - ) -> bool: - if allow_legacy_unscoped_cache_hits is not None: - return allow_legacy_unscoped_cache_hits - return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" - def _get_cache_logic(self, cached_response: Any): if cached_response is None: return cached_response @@ -210,8 +188,6 @@ class QdrantSemanticCache(BaseCache): } def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None: - if getattr(self, "allow_legacy_unscoped_cache_hits", False): - return data["filter"] = self._get_qdrant_cache_key_filter(key) def _ensure_cache_key_payload_index(self) -> None: @@ -236,14 +212,11 @@ class QdrantSemanticCache(BaseCache): ) def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: - # Legacy Qdrant semantic-cache points stored only prompt text and - # response. They cannot be reassigned to the generated LiteLLM cache key - # without risking cross-scope hits, so secure mode treats them as misses. + # Pre-isolation points stored only prompt + response with no cache-key + # payload field. Reassigning them to a caller's key would risk + # cross-scope hits, so they're treated as misses and re-populated on + # the next set_cache. cached_key = payload.get(self.CACHE_KEY_FIELD_NAME) - if cached_key is None and getattr( - self, "allow_legacy_unscoped_cache_hits", False - ): - return True return cached_key is not None and str(cached_key) == str(key) async def _get_async_embedding(self, prompt: str, **kwargs) -> Any: diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 35fed977cc..da9e7b1e58 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -36,9 +36,6 @@ class RedisSemanticCache(BaseCache): DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index" CACHE_KEY_FIELD_NAME: str = "litellm_cache_key" - ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR: str = ( - "LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS" - ) def __init__( self, @@ -49,7 +46,6 @@ class RedisSemanticCache(BaseCache): similarity_threshold: Optional[float] = None, embedding_model: str = "text-embedding-ada-002", index_name: Optional[str] = None, - allow_legacy_unscoped_cache_hits: Optional[bool] = None, **kwargs, ): """ @@ -91,10 +87,6 @@ class RedisSemanticCache(BaseCache): # While similarity: 1 = most similar, 0 = least similar self.distance_threshold = 1 - similarity_threshold self.embedding_model = embedding_model - self.allow_legacy_unscoped_cache_hits = ( - self._get_allow_legacy_unscoped_cache_hits(allow_legacy_unscoped_cache_hits) - ) - self._using_legacy_unscoped_index = False # Set up Redis connection if redis_url is None: @@ -125,14 +117,6 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) - @classmethod - def _get_allow_legacy_unscoped_cache_hits( - cls, allow_legacy_unscoped_cache_hits: Optional[bool] - ) -> bool: - if allow_legacy_unscoped_cache_hits is not None: - return allow_legacy_unscoped_cache_hits - return os.getenv(cls.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "").lower() == "true" - @classmethod def _cache_key_filterable_field(cls) -> Dict[str, str]: return { @@ -167,22 +151,6 @@ class RedisSemanticCache(BaseCache): if not _is_schema_mismatch(exc): raise - if self.allow_legacy_unscoped_cache_hits: - self._using_legacy_unscoped_index = True - print_verbose( - "Redis semantic-cache legacy unscoped hits are enabled via " - f"{self.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR}; reusing existing " - "index without cache-key isolation. Disable this after warming " - "the isolated semantic cache." - ) - return semantic_cache_cls( - name=index_name, - redis_url=redis_url, - vectorizer=cache_vectorizer, - distance_threshold=self.distance_threshold, - overwrite=False, - ) - isolated_index_name = f"{index_name}_isolated" print_verbose( "Redis semantic-cache existing index schema is not isolated; " @@ -223,11 +191,11 @@ class RedisSemanticCache(BaseCache): return Tag(self.CACHE_KEY_FIELD_NAME) == str(key) def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool: + # Pre-isolation entries with no ``litellm_cache_key`` field cannot be + # safely reassigned to a caller's scope and are treated as misses. cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) if isinstance(cached_key, bytes): cached_key = cached_key.decode("utf-8") - if cached_key is None and getattr(self, "_using_legacy_unscoped_index", False): - return True return cached_key is not None and str(cached_key) == str(key) def _get_ttl(self, **kwargs) -> Optional[int]: @@ -319,9 +287,9 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) value_str = str(value) - store_kwargs: Dict[str, Any] = {} - if not getattr(self, "_using_legacy_unscoped_index", False): - store_kwargs["filters"] = self._get_cache_filters(key) + store_kwargs: Dict[str, Any] = { + "filters": self._get_cache_filters(key), + } # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -357,11 +325,10 @@ class RedisSemanticCache(BaseCache): prompt = get_str_from_messages(messages) # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - check_kwargs: Dict[str, Any] = {"prompt": prompt} - if not getattr(self, "_using_legacy_unscoped_index", False): - check_kwargs["filter_expression"] = ( - self._get_cache_key_filter_expression(key) - ) + check_kwargs: Dict[str, Any] = { + "prompt": prompt, + "filter_expression": self._get_cache_key_filter_expression(key), + } results = self.llmcache.check(**check_kwargs) # Return None if no similar prompts found @@ -475,9 +442,8 @@ class RedisSemanticCache(BaseCache): store_kwargs: Dict[str, Any] = { "vector": prompt_embedding, + "filters": self._get_cache_filters(key), } - if not getattr(self, "_using_legacy_unscoped_index", False): - store_kwargs["filters"] = self._get_cache_filters(key) # Get TTL and store in Redis semantic cache ttl = self._get_ttl(**kwargs) @@ -522,11 +488,8 @@ class RedisSemanticCache(BaseCache): check_kwargs: Dict[str, Any] = { "prompt": prompt, "vector": prompt_embedding, + "filter_expression": self._get_cache_key_filter_expression(key), } - if not getattr(self, "_using_legacy_unscoped_index", False): - check_kwargs["filter_expression"] = ( - self._get_cache_key_filter_expression(key) - ) results = await self.llmcache.acheck(**check_kwargs) # handle results / cache hit diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 9b987a6d4f..949e6ccc29 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -208,76 +208,6 @@ def test_qdrant_semantic_cache_rejects_unscoped_cache_hit(): assert metadata["semantic-similarity"] == 0.0 -def test_qdrant_semantic_cache_allows_legacy_unscoped_hit_with_flag(monkeypatch): - monkeypatch.setenv("LITELLM_SEMANTIC_CACHE_ALLOW_LEGACY_UNSCOPED_HITS", "true") - - with ( - patch( - "litellm.llms.custom_httpx.http_handler._get_httpx_client" - ) as mock_sync_client, - patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"), - ): - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"result": {"exists": True}} - - mock_sync_client_instance = MagicMock() - mock_sync_client_instance.get.return_value = mock_response - mock_sync_client.return_value = mock_sync_client_instance - - from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache - - qdrant_cache = QdrantSemanticCache( - collection_name="test_collection", - qdrant_api_base="http://test.qdrant.local", - qdrant_api_key="test_key", - similarity_threshold=0.8, - ) - - mock_search_response = MagicMock() - mock_search_response.status_code = 200 - mock_search_response.json.return_value = { - "result": [ - { - "payload": { - "text": "What is the capital of France?", - "response": '{"id": "test-123"}', - }, - "score": 0.9, - } - ] - } - qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) - - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} - ): - metadata = {} - result = qdrant_cache.get_cache( - key="test_key", - messages=[{"content": "What is the capital of France?"}], - metadata=metadata, - ) - - assert result == {"id": "test-123"} - assert metadata["semantic-similarity"] == 0.9 - assert "filter" not in qdrant_cache.sync_client.post.call_args.kwargs["json"] - - -def test_qdrant_semantic_cache_legacy_mode_rejects_wrong_key_hit(): - from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache - - qdrant_cache = QdrantSemanticCache.__new__(QdrantSemanticCache) - qdrant_cache.allow_legacy_unscoped_cache_hits = True - - assert qdrant_cache._payload_matches_cache_key(payload={}, key="test_key") - assert not qdrant_cache._payload_matches_cache_key( - payload={QdrantSemanticCache.CACHE_KEY_FIELD_NAME: "other_key"}, - key="test_key", - ) - - def test_qdrant_semantic_cache_payload_index_failure_is_non_blocking(): from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index bebe1f757b..b50a35ef50 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -232,46 +232,6 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ] -def test_redis_semantic_cache_can_reuse_legacy_unscoped_index(monkeypatch): - fallback_cache_mock = MagicMock() - semantic_cache_mock = MagicMock( - side_effect=[ - ValueError("Existing index schema does not match"), - fallback_cache_mock, - ] - ) - custom_vectorizer_mock = MagicMock() - - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - monkeypatch.setenv("REDIS_HOST", "localhost") - monkeypatch.setenv("REDIS_PORT", "6379") - monkeypatch.setenv("REDIS_PASSWORD", "test_password") - monkeypatch.setenv( - RedisSemanticCache.ALLOW_LEGACY_UNSCOPED_HITS_ENV_VAR, "true" - ) - - redis_semantic_cache = RedisSemanticCache( - similarity_threshold=0.8, - index_name="existing_index", - ) - - assert redis_semantic_cache.llmcache is fallback_cache_mock - assert redis_semantic_cache._using_legacy_unscoped_index is True - assert semantic_cache_mock.call_count == 2 - assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index" - assert "filterable_fields" not in semantic_cache_mock.call_args_list[1].kwargs - - def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): fallback_cache_mock = MagicMock() semantic_cache_mock = MagicMock( @@ -372,11 +332,12 @@ def test_redis_semantic_cache_matches_bytes_cache_key(): ) -def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): +def test_redis_semantic_cache_rejects_pre_isolation_unscoped_hit(): + """Pre-isolation entries with no cache-key field cannot be safely + reassigned to a caller's scope and are treated as misses.""" from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) - redis_semantic_cache._using_legacy_unscoped_index = False cache_hit = { "prompt": "What is the capital of France?", @@ -388,12 +349,6 @@ def test_redis_semantic_cache_allows_unscoped_hit_only_in_legacy_mode(): key="test_key", ) - redis_semantic_cache._using_legacy_unscoped_index = True - assert redis_semantic_cache._cache_hit_matches_key( - cache_hit=cache_hit, - key="test_key", - ) - def test_redis_semantic_cache_builds_filter_expression(monkeypatch): class FakeTag: