[Performance] Performance improvements for /v1/messages route (#11421)

* fix: perf anthropic /v1/messages

* fix: perf anthropic /v1/messages

* fix: linting checks

* fix: linting checks

litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py

@@ -125,9 +125,9 @@ class AnthropicPassthroughLoggingHandler:
             litellm_model_response.id = logging_obj.litellm_call_id
             litellm_model_response.model = model
             logging_obj.model_call_details["model"] = model
-            logging_obj.model_call_details[
-                "custom_llm_provider"
-            ] = litellm.LlmProviders.ANTHROPIC.value
+            logging_obj.model_call_details["custom_llm_provider"] = (
+                litellm.LlmProviders.ANTHROPIC.value
+            )
             return kwargs
         except Exception as e:
             verbose_proxy_logger.exception(
@@ -210,10 +210,6 @@ class AnthropicPassthroughLoggingHandler:
                 if transformed_openai_chunk is not None:
                     all_openai_chunks.append(transformed_openai_chunk)
 
-                verbose_proxy_logger.debug(
-                    "all openai chunks= %s",
-                    json.dumps(all_openai_chunks, indent=4, default=str),
-                )
             except (StopIteration, StopAsyncIteration):
                 break
         complete_streaming_response = litellm.stream_chunk_builder(

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -1,5 +1,4 @@
 import asyncio
-import threading
 from datetime import datetime
 from typing import List, Optional
 
@@ -7,6 +6,7 @@ import httpx
 
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.proxy._types import PassThroughEndpointLoggingResultValues
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import StandardPassThroughResponseObject
@@ -122,15 +122,7 @@ class PassThroughStreamingHandler:
             standard_logging_response_object = StandardPassThroughResponseObject(
                 response=f"cannot parse chunks to standard response object. Chunks={all_chunks}"
             )
-        threading.Thread(
-            target=litellm_logging_obj.success_handler,
-            args=(
-                standard_logging_response_object,
-                start_time,
-                end_time,
-                False,
-            ),
-        ).start()
+
         await litellm_logging_obj.async_success_handler(
             result=standard_logging_response_object,
             start_time=start_time,
@@ -138,6 +130,17 @@ class PassThroughStreamingHandler:
             cache_hit=False,
             **kwargs,
         )
+        if litellm_logging_obj._should_run_sync_callbacks_for_async_calls() is False:
+            return
+
+        executor.submit(
+            litellm_logging_obj.success_handler,
+            result=standard_logging_response_object,
+            end_time=end_time,
+            cache_hit=False,
+            start_time=start_time,
+            **kwargs,
+        )
 
     @staticmethod
     def _convert_raw_bytes_to_str_lines(raw_bytes: List[bytes]) -> List[str]: