From 9fc8d1c82e16b27637c08ca3b54d4e3627d42ec0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 24 May 2025 16:29:53 -0700 Subject: [PATCH] [Chore]: feature flag aiohttp transport - users should opt into using aiohttp transport (#11132) * fix: feature flag aiohttp transport * fix: feature flag aiohttp transport --- litellm/__init__.py | 2 +- litellm/llms/custom_httpx/http_handler.py | 20 ++++++++++++++++++- .../health_endpoints/_health_endpoints.py | 3 +++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index bbb937a538..3ed894a21c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -300,7 +300,7 @@ custom_prometheus_metadata_labels: List[str] = [] priority_reservation: Optional[Dict[str, float]] = None ######## Networking Settings ######## -use_aiohttp_transport: bool = True +use_aiohttp_transport: bool = False force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. module_level_aclient = AsyncHTTPHandler( timeout=request_timeout, client_alias="module level aclient" diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index fe80e47d12..ee6cafbaa7 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -494,7 +494,7 @@ class AsyncHTTPHandler: # httpx_aiohttp is included in litellm docker images and pip when python 3.9+ is used ######################################################### if ( - litellm.use_aiohttp_transport + AsyncHTTPHandler._should_use_aiohttp_transport() and AsyncHTTPHandler.aiohttp_transport_exists() ): return AsyncHTTPHandler._create_aiohttp_transport( @@ -506,6 +506,24 @@ class AsyncHTTPHandler: ######################################################### return AsyncHTTPHandler._create_httpx_transport() + @staticmethod + def _should_use_aiohttp_transport() -> bool: + """ + This is feature flagged for now and is opt in as we roll out to all users. + + Controlled by either + - litellm.use_aiohttp_transport or os.getenv("USE_AIOHTTP_TRANSPORT") = "True" + """ + from litellm.secret_managers.main import str_to_bool + + if ( + str_to_bool(os.getenv("USE_AIOHTTP_TRANSPORT", "False")) + or litellm.use_aiohttp_transport + ): + verbose_logger.debug("Using AiohttpTransport...") + return True + return False + @staticmethod def _create_aiohttp_transport( ssl_verify: Optional[bool] = None, diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index b6bf0a3515..a283cf7866 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response, status import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( AlertType, CallInfo, @@ -547,6 +548,7 @@ async def health_readiness(): "cache": cache_type, "litellm_version": version, "success_callbacks": success_callback_names, + "use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(), **db_health_status, } else: @@ -556,6 +558,7 @@ async def health_readiness(): "cache": cache_type, "litellm_version": version, "success_callbacks": success_callback_names, + "use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(), } except Exception as e: raise HTTPException(status_code=503, detail=f"Service Unhealthy ({str(e)})")