From 727ab8dcc4b4bdbbbbf5f5baa9158ce4d912ccad Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:05:24 -0700 Subject: [PATCH 1/6] [Fix] Proxy: Break managed-resources import cycle on Python 3.13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Python 3.13 CCI smoke matrix surfaces a partially-initialized-module ImportError when loading the managed files hook chain: litellm.proxy.hooks/__init__ (mid-import) -> enterprise.enterprise_hooks -> litellm_enterprise.proxy.hooks.managed_files -> litellm.llms.base_llm.managed_resources.isolation -> litellm.proxy.management_endpoints.common_utils -> litellm.proxy.utils (re-enters litellm.proxy.hooks) The except ImportError block in hooks/__init__.py silently swallowed the failure, leaving managed_files unregistered and POST /files returning 500 "Managed files hook not found". Two-layer fix: - Inline the 3-line _user_has_admin_view check in isolation.py instead of importing it from litellm.proxy.management_endpoints.common_utils. litellm.llms.* should not depend on litellm.proxy.* — removing this layering violation breaks the cycle at its root. - Define PROXY_HOOKS and get_proxy_hook before the conditional enterprise import in litellm/proxy/hooks/__init__.py, so any future re-entry resolves the public names instead of hitting an ImportError on a partially-initialized module. Also fold in two unrelated CCI repairs surfaced in the same staging run: - tests/otel_tests/test_key_logging_callbacks.py: per-key gcs_bucket_name / gcs_path_service_account are now stripped by initialize_dynamic_callback_params, so the GCS client falls through to the env-only branch. Update the assertion to match the new "GCS_BUCKET_NAME is not set" message. - .circleci/config.yml: tests/pass_through_tests now resolves google-auth-library@10.x via the @google-cloud/vertexai 1.12.0 bump, which uses dynamic ESM imports Jest 29 cannot load without --experimental-vm-modules. Pass that flag in the Vertex JS test step. Adds tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py as a regression guard: managed_files / managed_vector_stores must register, and isolation.py must not transitively import litellm.proxy.utils. --- .circleci/config.yml | 2 +- .../base_llm/managed_resources/isolation.py | 10 +++- litellm/proxy/hooks/__init__.py | 30 +++++++----- .../otel_tests/test_key_logging_callbacks.py | 2 +- .../proxy/hooks/test_proxy_hooks_init.py | 47 +++++++++++++++++++ 5 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d2c4906ef6..a883c07021 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1935,7 +1935,7 @@ jobs: name: Run Vertex AI, Google AI Studio Node.js tests command: | cd tests/pass_through_tests - npx jest . --verbose + NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose no_output_timeout: 30m - run: name: Run tests diff --git a/litellm/llms/base_llm/managed_resources/isolation.py b/litellm/llms/base_llm/managed_resources/isolation.py index 4298d04462..340fd82242 100644 --- a/litellm/llms/base_llm/managed_resources/isolation.py +++ b/litellm/llms/base_llm/managed_resources/isolation.py @@ -11,8 +11,14 @@ unscoped query. from typing import Any, Dict, List, Optional -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + +def _user_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: + return user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) def build_list_page(items: List[Any], has_more: bool = False) -> Dict[str, Any]: diff --git a/litellm/proxy/hooks/__init__.py b/litellm/proxy/hooks/__init__.py index 790ebcd879..34505427d7 100644 --- a/litellm/proxy/hooks/__init__.py +++ b/litellm/proxy/hooks/__init__.py @@ -11,14 +11,10 @@ from .parallel_request_limiter import _PROXY_MaxParallelRequestsHandler from .parallel_request_limiter_v3 import _PROXY_MaxParallelRequestsHandler_v3 from .responses_id_security import ResponsesIDSecurity -### CHECK IF ENTERPRISE HOOKS ARE AVAILABLE #### - -try: - from enterprise.enterprise_hooks import ENTERPRISE_PROXY_HOOKS -except ImportError: - ENTERPRISE_PROXY_HOOKS = {} - -# List of all available hooks that can be enabled +# List of all available hooks that can be enabled. +# Defined before the enterprise import below so that any module re-imported +# transitively through `enterprise.enterprise_hooks` can resolve `PROXY_HOOKS` +# and `get_proxy_hook` from this partially-initialized module without circling. PROXY_HOOKS = { "max_budget_limiter": _PROXY_MaxBudgetLimiter, "parallel_request_limiter": _PROXY_MaxParallelRequestsHandler_v3, @@ -34,11 +30,6 @@ if os.getenv("LEGACY_MULTI_INSTANCE_RATE_LIMITING", "false").lower() == "true": PROXY_HOOKS["parallel_request_limiter"] = _PROXY_MaxParallelRequestsHandler -### update PROXY_HOOKS with ENTERPRISE_PROXY_HOOKS ### - -PROXY_HOOKS.update(ENTERPRISE_PROXY_HOOKS) - - def get_proxy_hook( hook_name: Union[ Literal[ @@ -58,3 +49,16 @@ def get_proxy_hook( f"Unknown hook: {hook_name}. Available hooks: {list(PROXY_HOOKS.keys())}" ) return PROXY_HOOKS[hook_name] + + +### CHECK IF ENTERPRISE HOOKS ARE AVAILABLE #### + +try: + from enterprise.enterprise_hooks import ENTERPRISE_PROXY_HOOKS +except ImportError: + ENTERPRISE_PROXY_HOOKS = {} + + +### update PROXY_HOOKS with ENTERPRISE_PROXY_HOOKS ### + +PROXY_HOOKS.update(ENTERPRISE_PROXY_HOOKS) diff --git a/tests/otel_tests/test_key_logging_callbacks.py b/tests/otel_tests/test_key_logging_callbacks.py index 96a13b8456..f3dbc10adb 100644 --- a/tests/otel_tests/test_key_logging_callbacks.py +++ b/tests/otel_tests/test_key_logging_callbacks.py @@ -64,6 +64,6 @@ async def test_key_logging_callbacks(): assert health_data["logging_callbacks"]["callbacks"] == ["gcs_bucket"] assert health_data["logging_callbacks"]["status"] == "unhealthy" assert ( - "Failed to load vertex credentials" + "GCS_BUCKET_NAME is not set in the environment" in health_data["logging_callbacks"]["details"] ) diff --git a/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py b/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py new file mode 100644 index 0000000000..23d8887a72 --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py @@ -0,0 +1,47 @@ +"""Regression guard for the enterprise hook registration / import cycle. + +Python 3.13 is stricter about partially-initialized modules and surfaces +cycles that Python 3.12 silently tolerated. The previous bug: + + litellm.proxy.hooks.__init__ + -> enterprise.enterprise_hooks + -> litellm_enterprise.proxy.hooks.managed_files + -> litellm.llms.base_llm.managed_resources.isolation + -> litellm.proxy.management_endpoints.common_utils + -> litellm.proxy.utils (re-enters litellm.proxy.hooks mid-init) + +silently swallowed the ImportError in `hooks/__init__.py`, leaving +``managed_files`` unregistered and the /files endpoint returning 500. +""" + +from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook + + +def test_managed_files_hook_registered(): + assert "managed_files" in PROXY_HOOKS + hook_cls = get_proxy_hook("managed_files") + assert hook_cls.__name__ == "_PROXY_LiteLLMManagedFiles" + + +def test_managed_vector_stores_hook_registered(): + assert "managed_vector_stores" in PROXY_HOOKS + hook_cls = get_proxy_hook("managed_vector_stores") + assert hook_cls.__name__ == "_PROXY_LiteLLMManagedVectorStores" + + +def test_isolation_module_does_not_pull_in_proxy_utils(): + """Layering guard: litellm.llms.* must not transitively import + litellm.proxy.utils, which would reintroduce the import cycle.""" + import importlib + import sys + + for mod in [ + "litellm.proxy.utils", + "litellm.proxy.management_endpoints.common_utils", + "litellm.llms.base_llm.managed_resources.isolation", + ]: + sys.modules.pop(mod, None) + + importlib.import_module("litellm.llms.base_llm.managed_resources.isolation") + assert "litellm.proxy.utils" not in sys.modules + assert "litellm.proxy.management_endpoints.common_utils" not in sys.modules From 8cac6c5bff927dee283debd165cc56cc91a34c0a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:13:31 -0700 Subject: [PATCH 2/6] [Fix] Proxy: Address Greptile feedback on hook-cycle PR - Move _user_has_admin_view to litellm.proxy._types as user_api_key_has_admin_view (single source of truth). common_utils.py and isolation.py both import from there now, removing the duplicated role-check that could silently diverge if new admin roles are added. - Add pytest.importorskip("litellm_enterprise") to the two regression tests that assert managed_files / managed_vector_stores are registered; those keys come from ENTERPRISE_PROXY_HOOKS so the tests would fail unconditionally in a checkout without the enterprise extra installed. --- .../llms/base_llm/managed_resources/isolation.py | 12 ++++-------- litellm/proxy/_types.py | 13 +++++++++++++ litellm/proxy/management_endpoints/common_utils.py | 8 +------- .../proxy/hooks/test_proxy_hooks_init.py | 4 ++++ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/litellm/llms/base_llm/managed_resources/isolation.py b/litellm/llms/base_llm/managed_resources/isolation.py index 340fd82242..62027f4272 100644 --- a/litellm/llms/base_llm/managed_resources/isolation.py +++ b/litellm/llms/base_llm/managed_resources/isolation.py @@ -11,14 +11,10 @@ unscoped query. from typing import Any, Dict, List, Optional -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - - -def _user_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: - return user_api_key_dict.user_role in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ) +from litellm.proxy._types import ( + UserAPIKeyAuth, + user_api_key_has_admin_view as _user_has_admin_view, +) def build_list_page(items: List[Any], has_more: bool = False) -> Dict[str, Any]: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index db7c487cdd..c6653a722d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2735,6 +2735,19 @@ class UserAPIKeyAuth( ) +def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: + """Return True if the caller's role grants unscoped read access to all + tenant resources (managed files, batches, vector stores, spend rows, etc). + + Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) + can use it without pulling in litellm.proxy.utils via management_endpoints. + """ + return user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + + class UserInfoResponse(LiteLLMPydanticObjectBase): user_id: Optional[str] user_info: Optional[Union[dict, BaseModel]] diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 8ad44b5300..a54c10bddc 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -16,6 +16,7 @@ from litellm.proxy._types import ( NewProjectRequest, UpdateProjectRequest, UserAPIKeyAuth, + user_api_key_has_admin_view as _user_has_admin_view, ) from litellm.proxy.utils import _premium_user_check @@ -24,13 +25,6 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging -def _user_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: - return ( - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN - or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY - ) - - def require_caller_user_id_for_non_admin( user_api_key_dict: UserAPIKeyAuth, ) -> str: diff --git a/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py b/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py index 23d8887a72..a6edd3db94 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_hooks_init.py @@ -14,16 +14,20 @@ silently swallowed the ImportError in `hooks/__init__.py`, leaving ``managed_files`` unregistered and the /files endpoint returning 500. """ +import pytest + from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook def test_managed_files_hook_registered(): + pytest.importorskip("litellm_enterprise") assert "managed_files" in PROXY_HOOKS hook_cls = get_proxy_hook("managed_files") assert hook_cls.__name__ == "_PROXY_LiteLLMManagedFiles" def test_managed_vector_stores_hook_registered(): + pytest.importorskip("litellm_enterprise") assert "managed_vector_stores" in PROXY_HOOKS hook_cls = get_proxy_hook("managed_vector_stores") assert hook_cls.__name__ == "_PROXY_LiteLLMManagedVectorStores" From 193907a4a3f54b63a963f3d231869a17b66b2235 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:16:59 -0700 Subject: [PATCH 3/6] [Fix] Lint: Mark _user_has_admin_view re-export in common_utils Ruff F401 flagged the aliased import as unused within common_utils.py because the name is consumed only by external modules (~15 callers across guardrails, spend tracking, MCP, agents, management endpoints). Add `# noqa: F401 re-exported` so the alias survives lint while keeping a single source of truth in litellm.proxy._types. --- litellm/proxy/management_endpoints/common_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index a54c10bddc..a43d15a580 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -16,7 +16,7 @@ from litellm.proxy._types import ( NewProjectRequest, UpdateProjectRequest, UserAPIKeyAuth, - user_api_key_has_admin_view as _user_has_admin_view, + user_api_key_has_admin_view as _user_has_admin_view, # noqa: F401 re-exported ) from litellm.proxy.utils import _premium_user_check From 8a1b6635fa4565a9e2d7e776a5450c6961302088 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:23:11 -0700 Subject: [PATCH 4/6] [Fix] Tests: Use master key for /otel-spans in test_chat_completion_check_otel_spans /otel-spans now requires proxy admin (returns 401 'Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route=/otel-spans' for non-admin callers). Switch the GET call to use the master key sk-1234 while keeping the generated key for the chat-completion request that produces the spans. --- tests/otel_tests/test_otel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/otel_tests/test_otel.py b/tests/otel_tests/test_otel.py index cf28d678de..a9ed4461a1 100644 --- a/tests/otel_tests/test_otel.py +++ b/tests/otel_tests/test_otel.py @@ -99,7 +99,8 @@ async def test_chat_completion_check_otel_spans(): await asyncio.sleep(3) - otel_spans = await get_otel_spans(session=session, key=key) + # /otel-spans requires proxy admin; use the master key. + otel_spans = await get_otel_spans(session=session, key="sk-1234") print("otel_spans: ", otel_spans) all_otel_spans = otel_spans["otel_spans"] From e6f524f95179661dfddba5c2d35d47e7de89f504 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:35:09 -0700 Subject: [PATCH 5/6] [Fix] Tests: Pick chat-completion OTEL trace by content, not recency The /otel-spans endpoint returns process-wide spans and tags most_recent_parent by max start_time. After tightening that route to proxy_admin (sk-1234), the GET /otel-spans request itself emits auth spans that beat the chat-completion spans on start_time, so most_recent_parent now points at the request's own auth trace (['postgres', 'postgres']) and the >=5-span assertion fails. Pick the chat-completion trace by content: it is the only trace whose span list is a superset of {postgres, redis, raw_gen_ai_request, batch_write_to_db}. Verified locally end-to-end against otel_test_config.yaml + OTEL_EXPORTER=in_memory: 3/3 runs green. --- tests/otel_tests/test_otel.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/otel_tests/test_otel.py b/tests/otel_tests/test_otel.py index a9ed4461a1..a0f58dd5b8 100644 --- a/tests/otel_tests/test_otel.py +++ b/tests/otel_tests/test_otel.py @@ -104,10 +104,24 @@ async def test_chat_completion_check_otel_spans(): print("otel_spans: ", otel_spans) all_otel_spans = otel_spans["otel_spans"] - most_recent_parent = str(otel_spans["most_recent_parent"]) - print("Most recent OTEL parent: ", most_recent_parent) - print("\n spans grouped by parent: ", otel_spans["spans_grouped_by_parent"]) - parent_trace_spans = otel_spans["spans_grouped_by_parent"][most_recent_parent] + spans_grouped_by_parent = otel_spans["spans_grouped_by_parent"] + print("\n spans grouped by parent: ", spans_grouped_by_parent) + + # The GET /otel-spans request itself produces auth spans that beat + # the chat-completion spans on start_time, so `most_recent_parent` + # points at the wrong trace. Pick the chat-completion trace by + # content: it's the one carrying the full set of expected markers. + chat_completion_markers = { + "postgres", + "redis", + "raw_gen_ai_request", + "batch_write_to_db", + } + parent_trace_spans = next( + spans + for spans in spans_grouped_by_parent.values() + if chat_completion_markers.issubset(spans) + ) print("Parent trace spans: ", parent_trace_spans) From 0976fbc6c40890a2433cfcd4c955f27570634e68 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 4 May 2026 20:54:54 -0700 Subject: [PATCH 6/6] [Fix] Tests: Restore /metrics access for prometheus test suite /metrics now requires auth by default; tests/otel_tests/test_prometheus.py makes 4+ unauthenticated GETs against http://0.0.0.0:4000/metrics, so every prometheus test in CI now fails the metric assertion. Set require_auth_for_metrics_endpoint: false in otel_test_config.yaml to opt out for this test job, which scrapes /metrics directly. Verified locally: 8/8 prometheus tests green (one flaky retry on test_proxy_success_metrics that pre-dates this PR). Also drop the -x stop-on-first-failure flag from the otel test command so all failures in the job surface in a single CI run rather than hiding behind whichever one trips first. --- .circleci/config.yml | 2 +- litellm/proxy/example_config_yaml/otel_test_config.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a883c07021..b91143f06f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1475,7 +1475,7 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/otel_tests --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: diff --git a/litellm/proxy/example_config_yaml/otel_test_config.yaml b/litellm/proxy/example_config_yaml/otel_test_config.yaml index fc506a792e..dc61286573 100644 --- a/litellm/proxy/example_config_yaml/otel_test_config.yaml +++ b/litellm/proxy/example_config_yaml/otel_test_config.yaml @@ -46,6 +46,9 @@ litellm_settings: cache: true callbacks: ["otel", "prometheus"] disable_end_user_cost_tracking_prometheus_only: True + # /metrics auth is on by default; tests/otel_tests/test_prometheus.py + # scrapes the endpoint without credentials, so opt out here. + require_auth_for_metrics_endpoint: False guardrails: - guardrail_name: "bedrock-pre-guard"