fix: remove StreamingResponse monkey-patch, bump uvicorn >=0.32.1

Uvicorn 0.31.x falsely advertised ASGI spec_version "2.4" without
implementing send() raising OSError on disconnect. Starlette trusted
this and skipped its disconnect listener, causing generators to run
forever. Uvicorn 0.32.1 corrected this to "2.3", restoring native
disconnect detection. The monkey-patch is no longer needed.

Also adds fallback_response cleanup in stream_with_fallbacks and
moves inline import anyio to module level in streaming_handler.

litellm/litellm_core_utils/streaming_handler.py

@@ -7,6 +7,7 @@ import time
 import traceback
 from typing import Any, Callable, Dict, List, Optional, Union, cast
 
+import anyio
 import httpx
 from pydantic import BaseModel
 
@@ -160,8 +161,6 @@ class CustomStreamWrapper:
             # Shield from anyio cancellation so cleanup awaits can complete.
             # Without this, CancelledError is thrown into every await during
             # task group cancellation, preventing HTTP connection release.
-            import anyio
-
             with anyio.CancelScope(shield=True):
                 try:
                     if hasattr(self.completion_stream, "aclose"):

litellm/proxy/common_request_processing.py

@@ -3,7 +3,6 @@ import json
 import logging
 import traceback
 from datetime import datetime
-from functools import partial
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -15,13 +14,10 @@ from typing import (
     Union,
 )
 
-import anyio
 import httpx
 import orjson
 from fastapi import HTTPException, Request, status
 from fastapi.responses import JSONResponse, Response, StreamingResponse
-from starlette._utils import collapse_excgroups
-from starlette.types import Receive, Scope, Send
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -142,39 +138,6 @@ def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict:
     return default_error
 
 
-async def _disconnect_aware_call(
-    self: StreamingResponse, scope: Scope, receive: Receive, send: Send
-) -> None:
-    """Patched StreamingResponse.__call__ that detects client disconnects.
-
-    Starlette >= 0.45.3 relies on ASGI spec 2.4 where send() should raise OSError
-    on client disconnect. Uvicorn does not implement this — send() silently returns.
-    This means async generators keep running forever after clients disconnect,
-    leaking upstream HTTP connections.
-
-    This restores the pre-0.45.3 behavior: a concurrent task listens for
-    http.disconnect and cancels the streaming task group when detected.
-
-    See: https://github.com/encode/starlette/pull/2732
-    See: https://github.com/encode/uvicorn/pull/2276
-    """
-    with collapse_excgroups():
-        async with anyio.create_task_group() as task_group:
-
-            async def wrap(func: Callable[[], Any]) -> None:
-                await func()
-                task_group.cancel_scope.cancel()
-
-            task_group.start_soon(wrap, partial(self.stream_response, send))
-            await wrap(partial(self.listen_for_disconnect, receive))
-
-    if self.background is not None:
-        await self.background()
-
-
-StreamingResponse.__call__ = _disconnect_aware_call  # type: ignore[assignment]
-
-
 async def create_response(
     generator: AsyncGenerator[str, None],
     media_type: str,

litellm/router.py

@@ -1497,6 +1497,7 @@ class Router:
                 return await self._async_generator.__anext__()
 
         async def stream_with_fallbacks():
+            fallback_response = None  # Track for cleanup in finally
             try:
                 async for item in model_response:
                     yield item
@@ -1596,13 +1597,17 @@ class Router:
                     )
                     raise fallback_error
             finally:
-                # Close the underlying stream to release the HTTP connection
+                # Close the underlying streams to release HTTP connections
                 # back to the connection pool when the generator is closed
                 # (e.g. on client disconnect).
-                # Shield from anyio cancellation so the await can complete.
-                if hasattr(model_response, "aclose"):
-                    with anyio.CancelScope(shield=True):
+                # Shield from anyio cancellation so the awaits can complete.
+                with anyio.CancelScope(shield=True):
+                    if hasattr(model_response, "aclose"):
                         await model_response.aclose()
+                    if fallback_response is not None and hasattr(
+                        fallback_response, "aclose"
+                    ):
+                        await fallback_response.aclose()
 
         return FallbackStreamWrapper(stream_with_fallbacks())
 

pyproject.toml

@@ -34,7 +34,7 @@ pydantic = "^2.5.0"
 jsonschema = ">=4.23.0,<5.0.0"
 numpydoc = {version = "*", optional = true} # used in utils.py
 
-uvicorn = {version = "^0.31.1", optional = true}
+uvicorn = {version = ">=0.32.1", optional = true}
 uvloop = {version = "^0.21.0", optional = true, markers="sys_platform != 'win32'"}
 gunicorn = {version = "^23.0.0", optional = true}
 fastapi = {version = ">=0.120.1", optional = true}