From de306cfcb3d6cefe06edeb32bffe418ecbc5da2a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Jun 2025 18:47:53 -0700 Subject: [PATCH] [Performance] Performance improvements for /v1/messages route (#11421) * fix: perf anthropic /v1/messages * fix: perf anthropic /v1/messages * fix: linting checks * fix: linting checks --- .../anthropic_passthrough_logging_handler.py | 10 +++----- .../streaming_handler.py | 23 +++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c4164daabe..0f1f215139 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -125,9 +125,9 @@ class AnthropicPassthroughLoggingHandler: litellm_model_response.id = logging_obj.litellm_call_id litellm_model_response.model = model logging_obj.model_call_details["model"] = model - logging_obj.model_call_details[ - "custom_llm_provider" - ] = litellm.LlmProviders.ANTHROPIC.value + logging_obj.model_call_details["custom_llm_provider"] = ( + litellm.LlmProviders.ANTHROPIC.value + ) return kwargs except Exception as e: verbose_proxy_logger.exception( @@ -210,10 +210,6 @@ class AnthropicPassthroughLoggingHandler: if transformed_openai_chunk is not None: all_openai_chunks.append(transformed_openai_chunk) - verbose_proxy_logger.debug( - "all openai chunks= %s", - json.dumps(all_openai_chunks, indent=4, default=str), - ) except (StopIteration, StopAsyncIteration): break complete_streaming_response = litellm.stream_chunk_builder( diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index d4260a0300..4efd43c9ff 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -1,5 +1,4 @@ import asyncio -import threading from datetime import datetime from typing import List, Optional @@ -7,6 +6,7 @@ import httpx from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.proxy._types import PassThroughEndpointLoggingResultValues from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import StandardPassThroughResponseObject @@ -122,15 +122,7 @@ class PassThroughStreamingHandler: standard_logging_response_object = StandardPassThroughResponseObject( response=f"cannot parse chunks to standard response object. Chunks={all_chunks}" ) - threading.Thread( - target=litellm_logging_obj.success_handler, - args=( - standard_logging_response_object, - start_time, - end_time, - False, - ), - ).start() + await litellm_logging_obj.async_success_handler( result=standard_logging_response_object, start_time=start_time, @@ -138,6 +130,17 @@ class PassThroughStreamingHandler: cache_hit=False, **kwargs, ) + if litellm_logging_obj._should_run_sync_callbacks_for_async_calls() is False: + return + + executor.submit( + litellm_logging_obj.success_handler, + result=standard_logging_response_object, + end_time=end_time, + cache_hit=False, + start_time=start_time, + **kwargs, + ) @staticmethod def _convert_raw_bytes_to_str_lines(raw_bytes: List[bytes]) -> List[str]: