tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-09 14:22:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm_passthrough_endpoints.py): working streaming on azure passthrough via router

Browse Source
This commit is contained in:
Krrish Dholakia
2025-10-06 13:56:30 -07:00
parent d7aab1d5de
commit 6f671d2bea
1 changed files with 40 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+40 -28
View File
@@ -840,39 +840,51 @@ async def azure_proxy_route(
if is_router_model:
request_body = await get_request_body(request)
is_streaming_request = is_passthrough_request_streaming(request_body)
result = cast(
httpx.Response,
await llm_router.allm_passthrough_route(
model=part,
method=request.method,
endpoint=endpoint,
request_query_params=request.query_params,
request_headers=dict(request.headers),
stream=request_body.get("stream", False),
content=None,
data=None,
files=None,
json=(
request_body
if request.headers.get("content-type") == "application/json"
else None
),
params=None,
headers=None,
cookies=None,
result = await llm_router.allm_passthrough_route(
model=part,
method=request.method,
endpoint=endpoint,
request_query_params=request.query_params,
request_headers=dict(request.headers),
stream=request_body.get("stream", False),
content=None,
data=None,
files=None,
json=(
request_body
if request.headers.get("content-type") == "application/json"
else None
),
params=None,
headers=None,
cookies=None,
)
if is_streaming_request:
return StreamingResponse(
content=result.aiter_bytes(),
status_code=result.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
custom_headers=None,
),
)
# Check if result is an async generator (from _async_streaming)
import inspect
if inspect.isasyncgen(result):
# Result is already an async generator, use it directly
return StreamingResponse(
content=result,
status_code=200,
headers={"content-type": "text/event-stream"},
)
else:
# Result is an httpx.Response, use aiter_bytes()
result = cast(httpx.Response, result)
return StreamingResponse(
content=result.aiter_bytes(),
status_code=result.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
custom_headers=None,
),
)
# Non-streaming response
result = cast(httpx.Response, result)
content = await result.aread()
return Response(
content=content,