From 33a051636d2f0b075dc09ae0ff3ff048f7848864 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 20:28:11 +0000 Subject: [PATCH] tests(llm_translation): add Redis cassette persister with 24h TTL Stores VCR cassettes in Redis under litellm:vcr:cassette: with a 24h expiry instead of YAML on disk. The TTL means each daily CI run starts with an aged-out cache, naturally re-records against live providers, and surfaces upstream API drift within a day without a manual `make` re-record sweep. Opt-in via LITELLM_VCR_REDIS=1; default behaviour is unchanged so local dev keeps the on-disk cassettes. before_record_response now drops non-2xx responses so a transient 5xx or 429 from a provider can't poison the cache for the rest of the TTL window. Vcr-marked tests bump litellm.num_retries to 3 during recording so provider-SDK exponential backoff kicks in on the cache-miss path. Tests cover the three surfaces we depend on in CI: serialize/deserialize roundtrip via the real vcrpy serializer, TTL is actually applied to saved keys, cache miss raises CassetteNotFoundError so vcrpy falls through to record mode, and 2xx-only filtering across the status-code matrix (2xx kept, 3xx/4xx/5xx dropped, with 429 and 503 explicitly pinned). --- tests/llm_translation/_vcr_redis_persister.py | 111 ++++++++++++++ tests/llm_translation/conftest.py | 53 ++++++- .../test_vcr_redis_persister.py | 137 ++++++++++++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 tests/llm_translation/_vcr_redis_persister.py create mode 100644 tests/llm_translation/test_vcr_redis_persister.py diff --git a/tests/llm_translation/_vcr_redis_persister.py b/tests/llm_translation/_vcr_redis_persister.py new file mode 100644 index 0000000000..88423393c5 --- /dev/null +++ b/tests/llm_translation/_vcr_redis_persister.py @@ -0,0 +1,111 @@ +"""Redis-backed cassette persister for vcrpy. + +Stores the same serialized cassette payload that ``FilesystemPersister`` +would write to disk, but under a Redis key with a 24h TTL. Cassettes +auto-expire so the next CI run after the rollover re-records against the +live provider, surfacing API drift within a day instead of waiting for a +human to refresh ``cassettes/*.yaml`` by hand. + +On a cache miss we raise ``CassetteNotFoundError``; vcrpy's record-mode +machinery catches that and falls through to a live HTTP call, which then +gets persisted via ``save_cassette``. Non-2xx responses are filtered out +upstream by ``conftest.before_record_response`` so a transient provider +failure can't poison the cache for 24h. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from vcr.persisters.filesystem import CassetteNotFoundError +from vcr.serialize import deserialize, serialize + +CASSETTE_TTL_SECONDS = 24 * 60 * 60 +REDIS_KEY_PREFIX = "litellm:vcr:cassette:" + + +def redis_key_for(cassette_path: str) -> str: + """Map a cassette file path to a stable Redis key. + + Uses the path relative to CWD so keys are stable across machines. + """ + rel = os.path.relpath(str(cassette_path)) + return f"{REDIS_KEY_PREFIX}{rel}" + + +def _build_default_client(): + import redis + + host = os.environ.get("REDIS_HOST") + if not host: + raise RuntimeError( + "REDIS_HOST is not set; cannot build Redis cassette persister" + ) + return redis.Redis( + host=host, + port=int(os.environ.get("REDIS_PORT", 6379)), + password=os.environ.get("REDIS_PASSWORD") or None, + socket_timeout=5, + socket_connect_timeout=5, + decode_responses=False, + ) + + +def make_redis_persister( + client: Optional[Any] = None, + ttl_seconds: int = CASSETTE_TTL_SECONDS, +): + """Build a vcrpy-compatible persister bound to a Redis client. + + The returned object exposes ``load_cassette`` / ``save_cassette`` and is + a drop-in replacement for ``vcr.persisters.filesystem.FilesystemPersister``. + Pass an explicit ``client`` in tests; production callers omit it and let + the persister build a client from ``REDIS_HOST`` / ``REDIS_PORT`` / + ``REDIS_PASSWORD``. + """ + redis_client = client if client is not None else _build_default_client() + + class _RedisPersister: + @staticmethod + def load_cassette(cassette_path, serializer): + data = redis_client.get(redis_key_for(cassette_path)) + if data is None: + raise CassetteNotFoundError() + if isinstance(data, bytes): + data = data.decode("utf-8") + return deserialize(data, serializer) + + @staticmethod + def save_cassette(cassette_path, cassette_dict, serializer): + data = serialize(cassette_dict, serializer) + payload = data.encode("utf-8") if isinstance(data, str) else data + redis_client.set( + redis_key_for(cassette_path), + payload, + ex=ttl_seconds, + ) + + return _RedisPersister + + +def filter_non_2xx_response(response): + """vcrpy ``before_record_response`` hook that drops non-2xx responses. + + Returning ``None`` tells vcrpy to skip persisting the response (see + ``vcr.cassette.Cassette.append``). This prevents transient 5xx/429 + failures from being baked into the cache for the rest of the TTL window. + """ + if not isinstance(response, dict): + return response + status = response.get("status") + code = None + if isinstance(status, dict): + code = status.get("code") + elif isinstance(status, int): + code = status + if code is None: + return response + if 200 <= int(code) < 300: + return response + return None diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index d944c9046c..0d9148a61a 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -14,10 +14,19 @@ import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +# Make sibling helper modules under tests/llm_translation/ importable regardless +# of the directory pytest is invoked from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import litellm import asyncio +from _vcr_redis_persister import ( # noqa: E402 (sibling module under conftest dir) + filter_non_2xx_response, + make_redis_persister, +) + # --------------------------------------------------------------------------- # VCR cassette infrastructure (pytest-recording) @@ -81,6 +90,16 @@ def _scrub_response(response): return response +def _before_record_response(response): + """Compose per-request scrubbing with the 2xx-only cache policy. + + Order matters: we scrub headers first so we don't leak request IDs even + on responses we end up dropping from the cassette mid-development. + """ + response = _scrub_response(response) + return filter_non_2xx_response(response) + + @pytest.fixture(scope="module") def vcr_config(): """Shared VCR config consumed by ``pytest-recording``. @@ -101,10 +120,22 @@ def vcr_config(): "query", "body", ), - "before_record_response": _scrub_response, + "before_record_response": _before_record_response, } +def pytest_recording_configure(config, vcr): + """Swap vcrpy's default filesystem persister for a Redis-backed one. + + Opt-in via ``LITELLM_VCR_REDIS=1`` so local dev keeps the YAML-on-disk + behaviour and CI (which sets the flag) gets a 24h-TTL cache that + auto-refreshes against live providers without manual ``make`` runs. + """ + if os.environ.get("LITELLM_VCR_REDIS") != "1": + return + vcr.register_persister(make_redis_persister()) + + # pytest-recording's default cassette dir is # ``/cassettes/``. Keep that — it gives every test its # own file and avoids name collisions across modules. @@ -187,6 +218,26 @@ def setup_and_teardown(event_loop): # Add event_loop as a dependency event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) +# Number of attempts a vcr-marked test gets when recording against a live +# provider. Replay-only runs never reach the network so this only matters on +# cache miss / record mode. Tenacity-style exponential backoff is provided by +# the underlying provider SDKs (openai, anthropic) when they see 429/5xx, so +# bumping num_retries propagates retry-with-backoff for free. +_VCR_RECORD_RETRIES = 3 + + +@pytest.fixture(autouse=True) +def _vcr_record_retries(setup_and_teardown, request): + """Configure record-time retries for ``@pytest.mark.vcr`` tests. + + Depends on ``setup_and_teardown`` so this runs *after* the per-test + ``importlib.reload(litellm)`` resets ``num_retries`` back to None. + """ + if request.node.get_closest_marker("vcr") is None: + return + litellm.num_retries = _VCR_RECORD_RETRIES + + def pytest_collection_modifyitems(config, items): # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py new file mode 100644 index 0000000000..e9ba57ac73 --- /dev/null +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -0,0 +1,137 @@ +"""Tests for the Redis-backed vcrpy cassette persister. + +These cover the three behaviours we actually rely on in CI: + +1. ``save_cassette`` followed by ``load_cassette`` returns the same + request/response pairs (roundtrip via the real vcrpy serializer). +2. Saved keys expire after ~24h so the cache auto-refreshes against live + providers without manual ``make`` runs. +3. ``load_cassette`` raises ``CassetteNotFoundError`` on a miss, so vcrpy's + record-mode machinery falls through to a live HTTP call instead of + silently matching against an empty cassette. + +We also pin the 2xx-only filter so a transient 5xx/429 from the provider +can't be baked into the cache for the rest of the TTL window. +""" + +from __future__ import annotations + +import os +import sys + +import fakeredis +import pytest +from vcr.persisters.filesystem import CassetteNotFoundError +from vcr.request import Request +from vcr.serializers import yamlserializer + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from _vcr_redis_persister import ( # noqa: E402 + CASSETTE_TTL_SECONDS, + filter_non_2xx_response, + make_redis_persister, + redis_key_for, +) + + +def _sample_cassette_dict(): + """Build a minimal cassette payload that exercises serialize/deserialize.""" + request = Request( + method="POST", + uri="https://api.anthropic.com/v1/messages", + body=b'{"model":"claude","messages":[{"role":"user","content":"hi"}]}', + headers={"content-type": "application/json"}, + ) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/json"]}, + # vcrpy stores response bodies as bytes; mirror that so the + # roundtrip assertion exercises real-world serialization shapes. + "body": {"string": b'{"id":"msg_1","type":"message"}'}, + } + return {"requests": [request], "responses": [response]} + + +def _persister_with_fake_redis(): + fake = fakeredis.FakeStrictRedis() + return fake, make_redis_persister(client=fake) + + +def test_save_then_load_roundtrips_cassette_content(): + """A saved cassette must come back from ``load_cassette`` identical to + what was put in. If serialize/deserialize ever drift (e.g. encoding bug) + every replay-mode test in the suite breaks; this catches it cheaply.""" + fake, persister = _persister_with_fake_redis() + cassette_path = "tests/llm_translation/cassettes/test_x/test_y.yaml" + + persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) + requests, responses = persister.load_cassette(cassette_path, yamlserializer) + + assert len(requests) == 1 + assert len(responses) == 1 + assert requests[0].method == "POST" + assert requests[0].uri == "https://api.anthropic.com/v1/messages" + assert responses[0]["status"]["code"] == 200 + assert responses[0]["body"]["string"] == b'{"id":"msg_1","type":"message"}' + + +def test_saved_key_has_24h_ttl(): + """The whole point of the Redis backend is that entries auto-expire after + 24h so each daily CI run re-records against live providers. If the TTL + isn't being applied, the cache never refreshes and we silently mask + upstream API drift.""" + fake, persister = _persister_with_fake_redis() + cassette_path = "tests/llm_translation/cassettes/test_x/test_ttl.yaml" + + persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) + + ttl = fake.ttl(redis_key_for(cassette_path)) + assert ttl > 0, "key was saved without an expiry — would never refresh" + assert ttl <= CASSETTE_TTL_SECONDS + assert ttl >= CASSETTE_TTL_SECONDS - 5 # allow tiny clock slack + + +def test_load_missing_key_raises_cassette_not_found(): + """Cache miss must surface as ``CassetteNotFoundError``. vcrpy's record + machinery catches that exception and falls through to the live HTTP + call; if we returned empty/None instead, vcrpy would treat it as a + cassette with zero matching requests and the test would fail with a + confusing ``CannotOverwriteExistingCassetteException``.""" + _, persister = _persister_with_fake_redis() + with pytest.raises(CassetteNotFoundError): + persister.load_cassette("never/recorded.yaml", yamlserializer) + + +@pytest.mark.parametrize( + ("status_code", "expect_dropped"), + [ + (200, False), + (201, False), + (204, False), + (299, False), + (300, True), + (400, True), + (401, True), + (404, True), + (429, True), # rate limit — must never be cached + (500, True), # transient 5xx — must never be cached + (502, True), + (503, True), + ], +) +def test_only_2xx_responses_are_cached(status_code, expect_dropped): + """Pin the cache-poisoning protection: a non-2xx must be dropped from + the cassette (returned as ``None`` from the hook) so a transient 429 + or 503 doesn't get pinned for the rest of the TTL window. 2xx + responses must pass through untouched.""" + response = { + "status": {"code": status_code, "message": "X"}, + "headers": {}, + "body": {"string": ""}, + } + result = filter_non_2xx_response(response) + if expect_dropped: + assert result is None + else: + assert result is response