From e37d1b0cb63d1a5d7f23918efa4e64c9e93a9166 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 23 Apr 2026 14:13:55 -0700 Subject: [PATCH 1/3] [Fix] Deflake spend tracking tests Two independent deflakes: 1. test_ui_view_spend_logs_unauthorized (unit) was returning 400 instead of 401/403 when earlier tests in the file left proxy-auth globals (prisma_client, master_key, user_custom_auth, general_settings, user_api_key_cache) in a state that let invalid tokens pass auth and fall through to the endpoint's own start_date/end_date validation. Add an autouse fixture that pins those globals to their import-time defaults for every test in the file. Harden the assertion to include response body so future flakes are diagnosable. 2. test_basic_spend_accuracy (CI job proxy_spend_accuracy_tests) depends on the Redis transaction buffer flushing spend to Postgres. The buffer uses a single global pod-lock key (cronjob_lock:db_spend_update_job) and a single global buffer list key. Pointing the proxy at the shared remote Redis means concurrent CI pipelines contend for the same lock and can drain each other's buffer into the wrong database. Add a start_redis reusable command that boots a per-job redis:7-alpine container (digest-pinned), and switch proxy_spend_accuracy_tests to REDIS_HOST=host.docker.internal:6379 so lock and buffer state are isolated per CI run. --- .circleci/config.yml | 29 +++++++++++++++---- .../test_spend_management_endpoints.py | 24 +++++++++++++-- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a59b7ef0d..e9b805fd45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,19 @@ commands: - wait_for_service: url: tcp://localhost:5432 timeout: "60" + start_redis: + description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." + steps: + - run: + name: Start Redis + command: | + docker run -d \ + --name redis-cache \ + -p 6379:6379 \ + redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 + - wait_for_service: + url: tcp://localhost:6379 + timeout: "60" setup_litellm_enterprise_pip: steps: - run: @@ -1775,6 +1788,7 @@ jobs: command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres + - start_redis - attach_workspace: at: ~/project - run: @@ -1784,15 +1798,18 @@ jobs: docker images | grep litellm-docker-database - run: name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace + # Point the proxy at the job-local Redis (start_redis) instead of the + # shared remote Redis. The Redis transaction buffer uses a single + # global pod-lock key (cronjob_lock:db_spend_update_job) and a single + # global buffer list (litellm_spend_update_buffer); sharing those + # across concurrent CI pipelines causes spend flushes to stall or + # land in the wrong DB, which is what makes this test flaky. command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ + -e REDIS_HOST=host.docker.internal \ + -e REDIS_PORT=6379 \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ @@ -1830,6 +1847,8 @@ jobs: command: | docker stop my-app docker rm my-app + docker stop redis-cache + docker rm redis-cache proxy_multi_instance_tests: machine: diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 1e2e398139..2370d5df30 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -422,6 +422,26 @@ def reset_router_callbacks(): litellm.logging_callback_manager._reset_all_callbacks() +@pytest.fixture(autouse=True) +def reset_proxy_auth_globals(monkeypatch): + """ + Pin proxy auth-related globals to a known baseline so tests don't inherit + leaked state (master_key, prisma_client, custom auth, cached tokens) from + earlier tests. Individual tests can still override via their own + monkeypatch calls — those run after this fixture and revert first. + """ + import litellm.proxy.proxy_server as ps + + monkeypatch.setattr(ps, "prisma_client", None) + monkeypatch.setattr(ps, "master_key", None) + monkeypatch.setattr(ps, "user_custom_auth", None) + monkeypatch.setattr(ps, "general_settings", {}) + try: + ps.user_api_key_cache.in_memory_cache.cache_dict.clear() + except AttributeError: + pass + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_user_id(client, monkeypatch): mock_spend_logs = [ @@ -1150,14 +1170,14 @@ async def test_ui_view_spend_logs_date_range_filter(client, monkeypatch): async def test_ui_view_spend_logs_unauthorized(client): # Test without authorization header response = client.get("/spend/logs/ui") - assert response.status_code == 401 or response.status_code == 403 + assert response.status_code in (401, 403), response.text # Test with invalid authorization response = client.get( "/spend/logs/ui", headers={"Authorization": "Bearer invalid-token"}, ) - assert response.status_code == 401 or response.status_code == 403 + assert response.status_code in (401, 403), response.text @pytest.mark.asyncio From 8adb3a6a8ff6bbd727e4d69fc588c3fa68862a61 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 23 Apr 2026 14:18:06 -0700 Subject: [PATCH 2/3] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9b805fd45..cb657fc2d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1844,6 +1844,9 @@ jobs: # Clean up first container - run: name: Stop and remove first container + - run: + name: Stop and remove first container + when: always command: | docker stop my-app docker rm my-app From 4af2b6735740b703563e78e8651d86149d0956de Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 23 Apr 2026 14:21:14 -0700 Subject: [PATCH 3/3] [Fix] Drop orphan teardown step from Greptile merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous commit from greptile-apps added a new `when: always` teardown step without removing the prior `name:`-only step, leaving a `- run` block with no `command:` — CircleCI config validation rejects that. Collapse back to a single teardown step that runs on success and failure. --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb657fc2d1..535dd1a9ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1841,9 +1841,6 @@ jobs: ls uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - # Clean up first container - - run: - name: Stop and remove first container - run: name: Stop and remove first container when: always