From 40edb16fb9991b326e6e02ddf7ddfcfd03b21091 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sun, 15 Mar 2026 17:25:01 -0700 Subject: [PATCH] Fix test isolation: run eager tiktoken tests in subprocesses The eager tiktoken tests were clearing all litellm modules from sys.modules and re-importing, creating new module objects with different class identities. This broke unittest.mock.patch for all subsequent tests on the same xdist worker. Running these tests in subprocesses provides perfect isolation. Fixes: test_metadata_passed_to_custom_callback_codex_models, test_oidc_github_success, test_oidc_google_cached, test_oidc_google_failure, test_encrypted_content_affinity_bypasses_rpm_limits, and 5 others. Co-Authored-By: Claude Opus 4.6 --- .../test_litellm/test_eager_tiktoken_load.py | 155 ++++++++---------- 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/tests/test_litellm/test_eager_tiktoken_load.py b/tests/test_litellm/test_eager_tiktoken_load.py index 33dd57fad8..8ea9836a5c 100644 --- a/tests/test_litellm/test_eager_tiktoken_load.py +++ b/tests/test_litellm/test_eager_tiktoken_load.py @@ -6,76 +6,83 @@ encoding is loaded at import time (pre-#18070 behavior) instead of lazy loading. This addresses issue #18659: VCR cassette creation broken by lazy loading. For now, this only affects encoding as it was the only reported issue. + +Tests that need to clear sys.modules and re-import litellm run in subprocesses +to avoid contaminating the test process's module graph (which breaks mock.patch +for all subsequent tests on the same xdist worker). """ -import os +import subprocess import sys +import textwrap + import pytest +def _run_python(script: str, env_override: dict | None = None) -> subprocess.CompletedProcess: + """Run a Python script in a subprocess and return the result.""" + import os + env = os.environ.copy() + # Remove the var so each test controls it explicitly + env.pop("LITELLM_DISABLE_LAZY_LOADING", None) + env.pop("TIKTOKEN_CACHE_DIR", None) + if env_override: + env.update(env_override) + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], + capture_output=True, + text=True, + env=env, + timeout=60, + ) + + def test_eager_loading_enabled(): """Test that encoding is loaded at import time when env var is set""" - # Set environment variable - os.environ["LITELLM_DISABLE_LAZY_LOADING"] = "1" - - # Clear any cached modules to ensure fresh import - modules_to_clear = [k for k in sys.modules.keys() if k.startswith("litellm")] - for module in modules_to_clear: - del sys.modules[module] - - # Import litellm - encoding should be loaded immediately - import litellm - - # Check that encoding is available (not lazy loaded) - assert hasattr(litellm, "encoding"), "Encoding should be available when eager loading is enabled" - - # Verify it's actually the encoding object - encoding = litellm.encoding - assert encoding is not None, "Encoding should not be None" - - # Test that it works - tokens = encoding.encode("Hello, world!") - assert len(tokens) > 0, "Encoding should work" + result = _run_python( + """ + import litellm + assert hasattr(litellm, "encoding"), "Encoding should be available when eager loading is enabled" + encoding = litellm.encoding + assert encoding is not None, "Encoding should not be None" + tokens = encoding.encode("Hello, world!") + assert len(tokens) > 0, "Encoding should work" + """, + env_override={"LITELLM_DISABLE_LAZY_LOADING": "1"}, + ) + assert result.returncode == 0, f"Subprocess failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" def test_eager_loading_env_var_values(): """Test that various env var values enable eager loading""" values = ["1", "true", "True", "TRUE", "yes", "Yes", "YES", "on", "On", "ON"] - for value in values: - os.environ["LITELLM_DISABLE_LAZY_LOADING"] = value - - # Clear modules - modules_to_clear = [k for k in sys.modules.keys() if k.startswith("litellm")] - for module in modules_to_clear: - del sys.modules[module] - - import litellm - assert hasattr(litellm, "encoding"), f"Encoding should be available for value: {value}" - encoding = litellm.encoding - tokens = encoding.encode("test") - assert len(tokens) > 0 + result = _run_python( + """ + import litellm + assert hasattr(litellm, "encoding"), "Encoding should be available" + encoding = litellm.encoding + tokens = encoding.encode("test") + assert len(tokens) > 0 + """, + env_override={"LITELLM_DISABLE_LAZY_LOADING": value}, + ) + assert result.returncode == 0, ( + f"Failed for value {value!r}:\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) def test_lazy_loading_default(): """Test that encoding is lazy loaded by default (when env var is not set)""" - # Remove environment variable if set - if "LITELLM_DISABLE_LAZY_LOADING" in os.environ: - del os.environ["LITELLM_DISABLE_LAZY_LOADING"] - - # Clear any cached modules - modules_to_clear = [k for k in sys.modules.keys() if k.startswith("litellm")] - for module in modules_to_clear: - del sys.modules[module] - - # Import litellm - encoding should NOT be loaded yet - import litellm - - # Encoding should be accessible via __getattr__ (lazy loading) - encoding = litellm.encoding # This triggers lazy loading - - # Verify it works - tokens = encoding.encode("Hello, world!") - assert len(tokens) > 0, "Encoding should work" + result = _run_python( + """ + import litellm + # Encoding should be accessible via __getattr__ (lazy loading) + encoding = litellm.encoding + tokens = encoding.encode("Hello, world!") + assert len(tokens) > 0, "Encoding should work" + """, + ) + assert result.returncode == 0, f"Subprocess failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" def test_tiktoken_cache_dir_set_on_lazy_load(): @@ -84,33 +91,15 @@ def test_tiktoken_cache_dir_set_on_lazy_load(): This ensures the local tiktoken cache is used instead of downloading from the internet. Regression test for issue #19768. """ - # Remove environment variables to ensure clean state - if "LITELLM_DISABLE_LAZY_LOADING" in os.environ: - del os.environ["LITELLM_DISABLE_LAZY_LOADING"] - if "TIKTOKEN_CACHE_DIR" in os.environ: - del os.environ["TIKTOKEN_CACHE_DIR"] - - # Clear any cached modules - modules_to_clear = [k for k in sys.modules.keys() if k.startswith("litellm")] - for module in modules_to_clear: - del sys.modules[module] - - # Import litellm fresh - import litellm - - # Access encoding (triggers lazy load) - _ = litellm.encoding - - # Verify TIKTOKEN_CACHE_DIR is now set and points to local tokenizers - assert "TIKTOKEN_CACHE_DIR" in os.environ, "TIKTOKEN_CACHE_DIR should be set after lazy loading encoding" - cache_dir = os.environ["TIKTOKEN_CACHE_DIR"] - assert "tokenizers" in cache_dir, f"TIKTOKEN_CACHE_DIR should point to tokenizers directory, got: {cache_dir}" - - -@pytest.fixture(autouse=True) -def cleanup_env(): - """Clean up environment variable after each test""" - yield - if "LITELLM_DISABLE_LAZY_LOADING" in os.environ: - del os.environ["LITELLM_DISABLE_LAZY_LOADING"] - + result = _run_python( + """ + import os + import litellm + # Access encoding (triggers lazy load) + _ = litellm.encoding + assert "TIKTOKEN_CACHE_DIR" in os.environ, "TIKTOKEN_CACHE_DIR should be set after lazy loading encoding" + cache_dir = os.environ["TIKTOKEN_CACHE_DIR"] + assert "tokenizers" in cache_dir, f"TIKTOKEN_CACHE_DIR should point to tokenizers directory, got: {cache_dir}" + """, + ) + assert result.returncode == 0, f"Subprocess failed:\nstdout: {result.stdout}\nstderr: {result.stderr}"