Merge pull request #22313 from BerriAI/tests/add_llmclientcache_regression_tests

Tests: add llmclientcache regression tests

tests/test_litellm/caching/test_llm_client_cache_e2e.py

@@ -0,0 +1,47 @@
+"""e2e tests: httpx clients obtained via get_async_httpx_client must remain
+usable after LLMClientCache evicts their cache entry."""
+
+import pytest
+
+import litellm
+from litellm.caching.llm_caching_handler import LLMClientCache
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+
+
+@pytest.fixture(autouse=True)
+def _tiny_client_cache(monkeypatch):
+    """Replace the global client cache with a size-1 cache so eviction
+    triggers on the second insert."""
+    cache = LLMClientCache(max_size_in_memory=1, default_ttl=600)
+    monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache)
+    yield cache
+
+
+@pytest.mark.asyncio
+async def test_evicted_client_is_not_closed():
+    """Get a client via get_async_httpx_client, evict it by caching a second
+    one, then verify the first client's transport is still open."""
+    client_a = get_async_httpx_client(llm_provider="provider_a")
+    # This evicts client_a from cache (capacity=1)
+    client_b = get_async_httpx_client(llm_provider="provider_b")
+
+    assert not client_a.client.is_closed
+    await client_a.client.aclose()
+    await client_b.client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_expired_client_is_not_closed():
+    """Get a client, expire it via TTL, then verify the client is still open."""
+    cache = litellm.in_memory_llm_clients_cache
+    client = get_async_httpx_client(llm_provider="provider_ttl")
+
+    # Force the entry to expire and trigger eviction
+    for key in list(cache.ttl_dict.keys()):
+        cache.ttl_dict[key] = 0
+        # Also fix the heap entry so evict_cache finds it
+        cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap]
+    cache.evict_cache()
+
+    assert not client.client.is_closed
+    await client.client.aclose()

tests/test_litellm/caching/test_redis_connection_pool.py

@@ -1,13 +1,15 @@
-"""Redis connection pool and LLMClientCache eviction tests."""
+"""
+Regression tests for Redis connection pool leak fixes (RC1-RC5).
+
+Tests are pure unit tests — no Redis server required.
+"""
 
 from unittest.mock import AsyncMock, MagicMock, patch
 
-import httpx
 import pytest
 import redis.asyncio as async_redis
 
 from litellm._redis import get_redis_async_client, get_redis_connection_pool
-from litellm.caching.llm_caching_handler import LLMClientCache
 
 
 def test_url_config_uses_passed_pool():
@@ -127,33 +129,3 @@ async def test_disconnect_idempotent():
     await cache.disconnect()  # should not raise
 
 
-# Regression: cache eviction must not close shared httpx clients (PR #22247)
-
-@pytest.mark.asyncio
-async def test_httpx_client_survives_capacity_eviction():
-    """Evicting an httpx client from LLMClientCache must NOT close it."""
-    cache = LLMClientCache(max_size_in_memory=1, default_ttl=600)
-    client = httpx.AsyncClient()
-
-    cache.set_cache("client_1", client)
-    # Exceed capacity — client_1 gets evicted
-    cache.set_cache("client_2", "other")
-
-    assert not client.is_closed
-    await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_httpx_client_survives_ttl_eviction():
-    """Evicting an httpx client via TTL expiry must NOT close it."""
-    cache = LLMClientCache(max_size_in_memory=200, default_ttl=600)
-    client = httpx.AsyncClient()
-
-    # TTL=0 so it expires immediately
-    cache.set_cache("client_1", client, ttl=0)
-    cache.evict_cache()
-
-    assert not client.is_closed
-    await client.aclose()
-
-