mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-15 16:19:45 +00:00
Multiple paths through _user_api_key_auth_builder returned a UserAPIKeyAuth without running common_checks(): OAuth2 token validation, OAuth2 proxy header hook, JWT admin shortcut, master_key path, pass-through custom headers, the /user/auth route, and the allow_requests_on_db_unavailable fallback. An operator-configured key model-access list, max_budget, team_blocked flag, or team model scope was therefore silently skipped on those paths. The HA-fallback token was worse: it was a full proxy-admin synthetic, so a DB outage granted full admin to every caller. Fix three root causes (VERIA-18): 1. Centralize common_checks in the user_api_key_auth wrapper. The builder paths no longer call it; the wrapper runs it once after the builder returns, for every path. Introduces _run_centralized_common_checks which gathers team/user/project/end_user/global_spend context in parallel via asyncio.gather. Preserves the existing custom_auth_run_common_checks opt-out for custom-auth deployments. 2. Narrow is_database_connection_error — drop the blanket PrismaError catch that routed data-layer errors (UniqueViolationError, etc.) into the HA fallback. Only real connectivity failures plus the no_db_connection marker now qualify. 3. DB-unavailable fallback issues an INTERNAL_USER token with user_id DB_UNAVAILABLE_FALLBACK_USER_ID instead of proxy-admin. An outage can no longer escalate an anonymous caller. JWT admin / master_key tokens still grant admin via a synthesized admin user_object (so non_proxy_admin_allowed_routes_check in common_checks recognizes them); other common_checks branches (team_blocked, team_model_access) now apply uniformly.
180 lines
5.6 KiB
Python
180 lines
5.6 KiB
Python
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
from fastapi import HTTPException, Request, status
|
|
from prisma import errors as prisma_errors
|
|
from prisma.errors import (
|
|
ClientNotConnectedError,
|
|
DataError,
|
|
ForeignKeyViolationError,
|
|
HTTPClientClosedError,
|
|
MissingRequiredValueError,
|
|
PrismaError,
|
|
RawQueryError,
|
|
RecordNotFoundError,
|
|
TableNotFoundError,
|
|
UniqueViolationError,
|
|
)
|
|
|
|
sys.path.insert(
|
|
0, os.path.abspath("../../..")
|
|
) # Adds the parent directory to the system path
|
|
|
|
from litellm._logging import verbose_proxy_logger
|
|
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
|
from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"prisma_error",
|
|
[
|
|
HTTPClientClosedError(),
|
|
ClientNotConnectedError(),
|
|
],
|
|
)
|
|
async def test_handle_authentication_error_db_unavailable_connectivity(prisma_error):
|
|
"""Transport-level / connectivity failures trigger the HA fallback."""
|
|
handler = UserAPIKeyAuthExceptionHandler()
|
|
|
|
mock_request = MagicMock()
|
|
with patch(
|
|
"litellm.proxy.proxy_server.general_settings",
|
|
{"allow_requests_on_db_unavailable": True},
|
|
):
|
|
result = await handler._handle_authentication_error(
|
|
prisma_error,
|
|
mock_request,
|
|
{},
|
|
"/test",
|
|
None,
|
|
"test-key",
|
|
)
|
|
assert result.key_name == "failed-to-connect-to-db"
|
|
assert result.token == "failed-to-connect-to-db"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"prisma_error",
|
|
[
|
|
PrismaError(),
|
|
DataError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
|
|
UniqueViolationError(
|
|
data={"user_facing_error": {"meta": {"table": "test_table"}}}
|
|
),
|
|
ForeignKeyViolationError(
|
|
data={"user_facing_error": {"meta": {"table": "test_table"}}}
|
|
),
|
|
MissingRequiredValueError(
|
|
data={"user_facing_error": {"meta": {"table": "test_table"}}}
|
|
),
|
|
RawQueryError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
|
|
TableNotFoundError(
|
|
data={"user_facing_error": {"meta": {"table": "test_table"}}}
|
|
),
|
|
RecordNotFoundError(
|
|
data={"user_facing_error": {"meta": {"table": "test_table"}}}
|
|
),
|
|
],
|
|
)
|
|
async def test_handle_authentication_error_data_layer_errors_do_not_fall_back(
|
|
prisma_error,
|
|
):
|
|
"""Data-layer PrismaError subclasses (UniqueViolation, RecordNotFound,
|
|
etc.) mean the DB IS reachable — they must propagate instead of
|
|
triggering the HA fallback, which would grant the restricted
|
|
INTERNAL_USER token to a request that should have returned 401."""
|
|
handler = UserAPIKeyAuthExceptionHandler()
|
|
|
|
mock_request = MagicMock()
|
|
with patch(
|
|
"litellm.proxy.proxy_server.general_settings",
|
|
{"allow_requests_on_db_unavailable": True},
|
|
):
|
|
with pytest.raises(ProxyException):
|
|
await handler._handle_authentication_error(
|
|
prisma_error,
|
|
mock_request,
|
|
{},
|
|
"/test",
|
|
None,
|
|
"test-key",
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_handle_authentication_error_budget_exceeded():
|
|
handler = UserAPIKeyAuthExceptionHandler()
|
|
|
|
# Mock request and other dependencies
|
|
mock_request = MagicMock()
|
|
mock_request_data = {}
|
|
mock_route = "/test"
|
|
mock_span = None
|
|
mock_api_key = "test-key"
|
|
|
|
# Test with budget exceeded error
|
|
with pytest.raises(ProxyException) as exc_info:
|
|
from litellm.exceptions import BudgetExceededError
|
|
|
|
budget_error = BudgetExceededError(
|
|
message="Budget exceeded", current_cost=100, max_budget=100
|
|
)
|
|
await handler._handle_authentication_error(
|
|
budget_error,
|
|
mock_request,
|
|
mock_request_data,
|
|
mock_route,
|
|
mock_span,
|
|
mock_api_key,
|
|
)
|
|
|
|
assert exc_info.value.type == ProxyErrorTypes.budget_exceeded
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_route_passed_to_post_call_failure_hook():
|
|
"""
|
|
This route is used by proxy track_cost_callback's async_post_call_failure_hook to check if the route is an LLM route
|
|
"""
|
|
handler = UserAPIKeyAuthExceptionHandler()
|
|
|
|
# Mock request and other dependencies
|
|
mock_request = MagicMock()
|
|
mock_request_data = {}
|
|
test_route = "/custom/route"
|
|
mock_span = None
|
|
mock_api_key = "test-key"
|
|
|
|
# Mock proxy_logging_obj.post_call_failure_hook
|
|
with patch(
|
|
"litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
|
|
new_callable=AsyncMock,
|
|
) as mock_post_call_failure_hook:
|
|
# Test with DB connection error
|
|
with patch(
|
|
"litellm.proxy.proxy_server.general_settings",
|
|
{"allow_requests_on_db_unavailable": False},
|
|
):
|
|
try:
|
|
await handler._handle_authentication_error(
|
|
PrismaError(),
|
|
mock_request,
|
|
mock_request_data,
|
|
test_route,
|
|
mock_span,
|
|
mock_api_key,
|
|
)
|
|
except Exception as e:
|
|
pass
|
|
asyncio.sleep(1)
|
|
# Verify post_call_failure_hook was called with the correct route
|
|
mock_post_call_failure_hook.assert_called_once()
|
|
call_args = mock_post_call_failure_hook.call_args[1]
|
|
assert call_args["user_api_key_dict"].request_route == test_route
|