Files
litellm/tests/test_litellm/proxy/db/test_exception_handler.py
T
Ishaan JaffandGitHub e0129710c8 fix(proxy): self-heal Prisma connection for auth and runtime (#21706)
* fix(proxy): add prisma reconnect primitive and db watchdog

* fix(proxy): start and stop prisma watchdog in lifecycle

* fix(auth): retry key lookup once after prisma reconnect

* test(proxy): add prisma self-heal watchdog coverage

* test(auth): cover reconnect-once behavior for key lookup

* refactor(auth): extract db reconnect helper and remove inline import

* fix(proxy): apply reconnect cooldown after attempt and add auth timeout path

* fix(auth): bound reconnect latency on key lookup path

* test(auth): assert reconnect timeout argument in key lookup

* test(proxy): verify reconnect cooldown timestamp set after attempt

* fix(proxy): harden prisma reconnect cycle semantics

* test(proxy): cover watchdog reconnect + timeout budget

* fix(proxy): bound watchdog probe and reconnect paths

* test(proxy): cover watchdog timeout and probe behavior

* fix(proxy): narrow prisma db connection error classification

* fix(proxy): add auth reconnect lock timeout budget

* fix(auth): pass lock timeout for db reconnect retries

* test(proxy): cover narrow prisma connection error detection

* test(proxy): add reconnect lock-timeout behavior coverage

* test(auth): assert reconnect lock timeout argument

* fix(proxy): avoid lock leak race in reconnect lock timeout path

* test(proxy): cover reconnect lock-timeout race cleanup
2026-02-20 18:11:36 -08:00

162 lines
4.9 KiB
Python

import asyncio
import json
import os
import sys
from unittest.mock import MagicMock, patch
import pytest
from fastapi import HTTPException, Request, status
from prisma import errors as prisma_errors
from prisma.errors import (
ClientNotConnectedError,
DataError,
ForeignKeyViolationError,
HTTPClientClosedError,
MissingRequiredValueError,
PrismaError,
RawQueryError,
RecordNotFoundError,
TableNotFoundError,
UniqueViolationError,
)
sys.path.insert(
0, os.path.abspath("../../..")
) # Adds the parent directory to the system path
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
# Test is_database_connection_error method
@pytest.mark.parametrize(
"prisma_error",
[
HTTPClientClosedError(),
ClientNotConnectedError(),
PrismaError("can't reach database server"),
PrismaError("connection refused"),
PrismaError("timed out while connecting"),
],
)
def test_is_database_connection_error_prisma_connection_errors(prisma_error):
"""
Test that only Prisma connection-related errors are considered DB connection errors.
"""
assert PrismaDBExceptionHandler.is_database_connection_error(prisma_error) == True
@pytest.mark.parametrize(
"prisma_error",
[
PrismaError(),
PrismaError("validation failed on query"),
DataError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
UniqueViolationError(
data={"user_facing_error": {"meta": {"table": "test_table"}}}
),
ForeignKeyViolationError(
data={"user_facing_error": {"meta": {"table": "test_table"}}}
),
MissingRequiredValueError(
data={"user_facing_error": {"meta": {"table": "test_table"}}}
),
RawQueryError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
TableNotFoundError(
data={"user_facing_error": {"meta": {"table": "test_table"}}}
),
RecordNotFoundError(
data={"user_facing_error": {"meta": {"table": "test_table"}}}
),
],
)
def test_is_database_connection_error_non_connection_prisma_errors(prisma_error):
assert PrismaDBExceptionHandler.is_database_connection_error(prisma_error) == False
def test_is_database_connection_generic_errors():
"""
Test non-Prisma error cases for database connection checking
"""
assert (
PrismaDBExceptionHandler.is_database_connection_error(
Exception("Regular error")
)
== False
)
# Test with ProxyException (DB connection)
db_proxy_exception = ProxyException(
message="DB Connection Error",
type=ProxyErrorTypes.no_db_connection,
param="test-param",
)
assert (
PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception)
== True
)
# Test with non-DB error
regular_exception = Exception("Regular error")
assert (
PrismaDBExceptionHandler.is_database_connection_error(regular_exception)
== False
)
# Test should_allow_request_on_db_unavailable method
@patch(
"litellm.proxy.proxy_server.general_settings",
{"allow_requests_on_db_unavailable": True},
)
def test_should_allow_request_on_db_unavailable_true():
assert PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() == True
@patch(
"litellm.proxy.proxy_server.general_settings",
{"allow_requests_on_db_unavailable": False},
)
def test_should_allow_request_on_db_unavailable_false():
assert PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() == False
@patch(
"litellm.proxy.proxy_server.general_settings",
{"allow_requests_on_db_unavailable": True},
)
def test_handle_db_exception_with_connection_error():
"""
Test that DB connection errors are handled gracefully when allow_requests_on_db_unavailable is True
"""
db_error = ClientNotConnectedError()
result = PrismaDBExceptionHandler.handle_db_exception(db_error)
assert result is None
@patch(
"litellm.proxy.proxy_server.general_settings",
{"allow_requests_on_db_unavailable": False},
)
def test_handle_db_exception_raises_error():
"""
Test that DB connection errors are raised when allow_requests_on_db_unavailable is False
"""
db_error = ClientNotConnectedError()
with pytest.raises(ClientNotConnectedError):
PrismaDBExceptionHandler.handle_db_exception(db_error)
def test_handle_db_exception_with_non_db_error():
"""
Test that non-DB errors are always raised regardless of allow_requests_on_db_unavailable setting
"""
regular_error = litellm.BudgetExceededError(
current_cost=10,
max_budget=10,
)
with pytest.raises(litellm.BudgetExceededError):
PrismaDBExceptionHandler.handle_db_exception(regular_error)