From a4f94b241b6f106dbba73ce5244891b426335e63 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Fri, 13 Mar 2026 04:32:36 +0100 Subject: [PATCH 1/3] fix(proxy): prevent duplicate callback logs for pass-through endpoint failures Pass-through endpoint failures fired both async_failure_handler and async_post_call_failure_hook, causing duplicate logs in callback integrations. Add pass-through guards to the failure path, matching the existing success path behavior. --- litellm/litellm_core_utils/litellm_logging.py | 5 +- .../pass_through_endpoints.py | 8 ++ litellm/proxy/utils.py | 19 ++- tests/proxy_unit_tests/test_proxy_utils.py | 131 ++++++++++++++++++ .../test_litellm_logging.py | 65 +++++++++ 5 files changed, 223 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e22d057bb6..24fcbde1a0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2920,7 +2920,10 @@ class Logging(LiteLLMLoggingBaseClass): callback_func=callback, ) if ( - isinstance(callback, CustomLogger) and is_sync_request + isinstance(callback, CustomLogger) + and is_sync_request + and self.call_type + != CallTypes.pass_through.value # pass-through endpoints call async_log_failure_event ): # custom logger class callback.log_failure_event( start_time=start_time, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 9e287c2bec..871d9848ab 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -961,6 +961,10 @@ async def pass_through_request( # noqa: PLR0915 if kwargs: for key, value in kwargs.items(): request_payload[key] = value + # Ensure the original logging_obj is in request_payload so + # _handle_logging_proxy_only_error reuses it (preserving call_type) + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj if ( "model" not in request_payload @@ -1703,6 +1707,8 @@ async def websocket_passthrough_request( # noqa: PLR0915 if kwargs: for key, value in kwargs.items(): request_payload[key] = value + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj # Log the connection failure using the same pattern as HTTP await proxy_logging_obj.post_call_failure_hook( @@ -1729,6 +1735,8 @@ async def websocket_passthrough_request( # noqa: PLR0915 if kwargs: for key, value in kwargs.items(): request_payload[key] = value + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj # Log the unexpected error using the same pattern as HTTP await proxy_logging_obj.post_call_failure_hook( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b9a1bfb906..695f75b86e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1849,7 +1849,7 @@ class ProxyLogging: for k, v in request_data.items(): if k in litellm_param_keys: _litellm_params[k] = v - elif k != "model" and k != "user": + elif k not in ("model", "user", "litellm_logging_obj"): _optional_params[k] = v litellm_logging_obj.update_environment_variables( @@ -1865,20 +1865,31 @@ class ProxyLogging: ): input = request_data["messages"] litellm_logging_obj.model_call_details["messages"] = input - litellm_logging_obj.call_type = CallTypes.acompletion.value + if litellm_logging_obj.call_type != CallTypes.pass_through.value: + litellm_logging_obj.call_type = CallTypes.acompletion.value elif "prompt" in request_data and isinstance(request_data["prompt"], str): input = request_data["prompt"] litellm_logging_obj.model_call_details["prompt"] = input - litellm_logging_obj.call_type = CallTypes.atext_completion.value + if litellm_logging_obj.call_type != CallTypes.pass_through.value: + litellm_logging_obj.call_type = CallTypes.atext_completion.value elif "input" in request_data and isinstance(request_data["input"], list): input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input - litellm_logging_obj.call_type = CallTypes.aembedding.value + if litellm_logging_obj.call_type != CallTypes.pass_through.value: + litellm_logging_obj.call_type = CallTypes.aembedding.value litellm_logging_obj.pre_call( input=input, api_key="", ) + # For pass-through endpoints, skip async_failure_handler and + # failure_handler here. The callback loop in post_call_failure_hook + # will call async_post_call_failure_hook on each CustomLogger, + # which handles logging. Firing both paths would produce duplicate + # entries in Datadog, Arize, and other callback integrations. + if litellm_logging_obj.call_type == CallTypes.pass_through.value: + return + # log the custom exception await litellm_logging_obj.async_failure_handler( exception=original_exception, diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index b2ed4f9103..2f82e1d194 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2385,3 +2385,134 @@ async def test_during_call_hook_parallel_execution_with_error(): assert "Guardrail violation detected!" in str(exc_info.value) finally: litellm.callbacks = original_callbacks + + +@pytest.mark.asyncio +async def test_handle_logging_proxy_only_error_preserves_pass_through_call_type(): + """Ensure _handle_logging_proxy_only_error does not overwrite call_type + when the logging object is already marked as pass_through_endpoint. + """ + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import ProxyLogging + from litellm.types.utils import CallTypes + + logging_obj = Logging( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + + request_data = { + "litellm_logging_obj": logging_obj, + "messages": [{"role": "user", "content": "test"}], + "model": "claude-3-5-sonnet", + } + + cache = DualCache() + proxy_logging = ProxyLogging(user_api_key_cache=cache) + + with patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock): + with patch.object(logging_obj, "failure_handler"): + await proxy_logging._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", token="test_token" + ), + original_exception=Exception("test error"), + ) + + assert logging_obj.call_type == CallTypes.pass_through.value + + +@pytest.mark.asyncio +async def test_litellm_logging_obj_excluded_from_optional_params(): + """Ensure litellm_logging_obj is excluded from _optional_params to prevent + circular references in model_call_details. + """ + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import ProxyLogging + + logging_obj = Logging( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + + request_data = { + "litellm_logging_obj": logging_obj, + "messages": [{"role": "user", "content": "test"}], + "model": "claude-3-5-sonnet", + } + + cache = DualCache() + proxy_logging = ProxyLogging(user_api_key_cache=cache) + + with patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock): + with patch.object(logging_obj, "failure_handler"): + await proxy_logging._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", token="test_token" + ), + original_exception=Exception("test error"), + ) + + assert "litellm_logging_obj" not in logging_obj.model_call_details + + +@pytest.mark.asyncio +async def test_handle_logging_proxy_only_error_skips_handlers_for_pass_through(): + """Ensure _handle_logging_proxy_only_error skips async_failure_handler and + failure_handler for pass-through endpoint errors, so only + async_post_call_failure_hook fires (avoiding duplicate logs). + + Regression test for duplicate Datadog/Arize logs on pass-through failures. + """ + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import ProxyLogging + from litellm.types.utils import CallTypes + + logging_obj = Logging( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + + cache = DualCache() + proxy_logging = ProxyLogging(user_api_key_cache=cache) + + request_data = { + "litellm_logging_obj": logging_obj, + "messages": [{"role": "user", "content": "test"}], + "model": "claude-3-5-sonnet", + } + + with patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async: + with patch.object(logging_obj, "failure_handler") as mock_sync: + await proxy_logging._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", token="test_token" + ), + original_exception=Exception("test error"), + ) + + # Neither handler should fire for pass-through requests + mock_async.assert_not_called() + mock_sync.assert_not_called() + assert logging_obj.call_type == CallTypes.pass_through.value diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index f4aeb27b31..482ee049fa 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1833,3 +1833,68 @@ def test_function_setup_empty_metadata_falls_back_to_litellm_metadata(): assert metadata is not None assert metadata.get("user_api_key_hash") == "sk-hashed-empty-test" assert metadata.get("user_api_key_team_id") == "team-empty-test" + + +def test_failure_handler_skips_sync_callbacks_for_pass_through_requests(logging_obj): + """Ensure sync failure callbacks are skipped for pass-through endpoint requests. + + Regression test for duplicate Datadog/Arize logs on pass-through endpoint failures. + The async_failure_handler fires async_log_failure_event; the sync failure_handler + must NOT also fire log_failure_event for pass-through requests. + """ + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import CallTypes + + class DummyLogger(CustomLogger): + pass + + logging_obj.call_type = CallTypes.pass_through.value + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + dummy_logger = DummyLogger() + dummy_logger.log_failure_event = MagicMock() + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[dummy_logger], + ): + logging_obj.failure_handler( + exception=Exception("test error"), + traceback_exception="", + ) + + dummy_logger.log_failure_event.assert_not_called() + + +@pytest.mark.parametrize("call_type", ["completion", "acompletion"]) +def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests( + logging_obj, call_type +): + """Ensure sync failure callbacks still fire for normal (non-pass-through) requests.""" + from litellm.integrations.custom_logger import CustomLogger + + class DummyLogger(CustomLogger): + pass + + logging_obj.call_type = call_type + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + dummy_logger = DummyLogger() + dummy_logger.log_failure_event = MagicMock() + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[dummy_logger], + ): + logging_obj.failure_handler( + exception=Exception("test error"), + traceback_exception="", + ) + + dummy_logger.log_failure_event.assert_called_once() From 583bdb279b36a6ad1dc1113b2ea63cc0f4c05380 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Fri, 13 Mar 2026 04:53:50 +0100 Subject: [PATCH 2/3] address greptile review: move early return before pre_call, trim comments --- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../pass_through_endpoints.py | 2 -- litellm/proxy/utils.py | 13 +++++-------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 24fcbde1a0..d091a263df 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2923,7 +2923,7 @@ class Logging(LiteLLMLoggingBaseClass): isinstance(callback, CustomLogger) and is_sync_request and self.call_type - != CallTypes.pass_through.value # pass-through endpoints call async_log_failure_event + != CallTypes.pass_through.value ): # custom logger class callback.log_failure_event( start_time=start_time, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 871d9848ab..0aa9968520 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -961,8 +961,6 @@ async def pass_through_request( # noqa: PLR0915 if kwargs: for key, value in kwargs.items(): request_payload[key] = value - # Ensure the original logging_obj is in request_payload so - # _handle_logging_proxy_only_error reuses it (preserving call_type) if logging_obj is not None: request_payload["litellm_logging_obj"] = logging_obj diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 695f75b86e..b0d709abbb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1877,19 +1877,16 @@ class ProxyLogging: litellm_logging_obj.model_call_details["input"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: litellm_logging_obj.call_type = CallTypes.aembedding.value + # Pass-through endpoints are logged via the callback loop's + # async_post_call_failure_hook — skip pre_call and failure handlers. + if litellm_logging_obj.call_type == CallTypes.pass_through.value: + return + litellm_logging_obj.pre_call( input=input, api_key="", ) - # For pass-through endpoints, skip async_failure_handler and - # failure_handler here. The callback loop in post_call_failure_hook - # will call async_post_call_failure_hook on each CustomLogger, - # which handles logging. Firing both paths would produce duplicate - # entries in Datadog, Arize, and other callback integrations. - if litellm_logging_obj.call_type == CallTypes.pass_through.value: - return - # log the custom exception await litellm_logging_obj.async_failure_handler( exception=original_exception, From bfcba21564fd993fcc769fe22bdce7674b1ee102 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Fri, 13 Mar 2026 05:01:50 +0100 Subject: [PATCH 3/3] pop litellm_logging_obj from request_data before callback loop --- litellm/proxy/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b0d709abbb..e2df0acd21 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1729,6 +1729,9 @@ class ProxyLogging: original_exception=original_exception, ) + # Remove before callbacks iterate — not serialisable + request_data.pop("litellm_logging_obj", None) + # Track the first HTTPException returned or raised by any callback transformed_exception: Optional[HTTPException] = None