Merge pull request #26349 from BerriAI/litellm_deflakeSpendTests

[Fix] Deflake spend tracking tests

.circleci/config.yml

@@ -98,6 +98,19 @@ commands:
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
+  start_redis:
+    description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys."
+    steps:
+      - run:
+          name: Start Redis
+          command: |
+            docker run -d \
+              --name redis-cache \
+              -p 6379:6379 \
+              redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
+      - wait_for_service:
+          url: tcp://localhost:6379
+          timeout: "60"
   setup_litellm_enterprise_pip:
     steps:
       - run:
@@ -1591,6 +1604,7 @@ jobs:
           command: |
             uv sync --frozen --all-groups --all-extras --python 3.12
       - start_postgres
+      - start_redis
       - attach_workspace:
           at: ~/project
       - run:
@@ -1600,15 +1614,18 @@ jobs:
             docker images | grep litellm-docker-database
       - run:
           name: Run Docker container
-          # intentionally give bad redis credentials here
-          # the OTEL test - should get this as a trace
+          # Point the proxy at the job-local Redis (start_redis) instead of the
+          # shared remote Redis. The Redis transaction buffer uses a single
+          # global pod-lock key (cronjob_lock:db_spend_update_job) and a single
+          # global buffer list (litellm_spend_update_buffer); sharing those
+          # across concurrent CI pipelines causes spend flushes to stall or
+          # land in the wrong DB, which is what makes this test flaky.
           command: |
             docker run -d \
               -p 4000:4000 \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e REDIS_HOST=$REDIS_HOST \
-              -e REDIS_PASSWORD=$REDIS_PASSWORD \
-              -e REDIS_PORT=$REDIS_PORT \
+              -e REDIS_HOST=host.docker.internal \
+              -e REDIS_PORT=6379 \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
@@ -1638,12 +1655,14 @@ jobs:
           command: |
             uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 15m
-            # Clean up first container
       - run:
           name: Stop and remove first container
+          when: always
           command: |
             docker stop my-app
             docker rm my-app
+            docker stop redis-cache
+            docker rm redis-cache
 
   proxy_multi_instance_tests:
     machine:

tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py

@@ -422,6 +422,26 @@ def reset_router_callbacks():
     litellm.logging_callback_manager._reset_all_callbacks()
 
 
+@pytest.fixture(autouse=True)
+def reset_proxy_auth_globals(monkeypatch):
+    """
+    Pin proxy auth-related globals to a known baseline so tests don't inherit
+    leaked state (master_key, prisma_client, custom auth, cached tokens) from
+    earlier tests. Individual tests can still override via their own
+    monkeypatch calls — those run after this fixture and revert first.
+    """
+    import litellm.proxy.proxy_server as ps
+
+    monkeypatch.setattr(ps, "prisma_client", None)
+    monkeypatch.setattr(ps, "master_key", None)
+    monkeypatch.setattr(ps, "user_custom_auth", None)
+    monkeypatch.setattr(ps, "general_settings", {})
+    try:
+        ps.user_api_key_cache.in_memory_cache.cache_dict.clear()
+    except AttributeError:
+        pass
+
+
 @pytest.mark.asyncio
 async def test_ui_view_spend_logs_with_user_id(client, monkeypatch):
     mock_spend_logs = [
@@ -1150,14 +1170,14 @@ async def test_ui_view_spend_logs_date_range_filter(client, monkeypatch):
 async def test_ui_view_spend_logs_unauthorized(client):
     # Test without authorization header
     response = client.get("/spend/logs/ui")
-    assert response.status_code == 401 or response.status_code == 403
+    assert response.status_code in (401, 403), response.text
 
     # Test with invalid authorization
     response = client.get(
         "/spend/logs/ui",
         headers={"Authorization": "Bearer invalid-token"},
     )
-    assert response.status_code == 401 or response.status_code == 403
+    assert response.status_code in (401, 403), response.text
 
 
 @pytest.mark.asyncio