diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md index c853a860de..15a838bb7d 100644 --- a/docs/my-website/docs/realtime.md +++ b/docs/my-website/docs/realtime.md @@ -110,7 +110,88 @@ ws.on("error", function handleError(error) { }); ``` -## Logging +## Guardrails + +You can apply [LiteLLM guardrails](https://docs.litellm.ai/docs/proxy/guardrails/quick_start) to realtime sessions. + +### Set guardrails on a key or team + +The easiest production setup — attach guardrails to a virtual key or team so they always apply automatically, without any client-side changes. + +See [Virtual Keys → Guardrails](https://docs.litellm.ai/docs/proxy/virtual_keys#guardrails) and [Teams → Guardrails](https://docs.litellm.ai/docs/proxy/team_budgets). + +### Pass guardrails dynamically (easy testing) + +Pass `guardrails` as a query param when opening the WebSocket. +Useful for testing guardrails without modifying key/team config. + +```js +// node test.js +const WebSocket = require("ws"); + +const guardrails = ["your-guardrail-name"]; // comma-separated list +const url = `ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=${guardrails.join(",")}`; + +const ws = new WebSocket(url, { + headers: { + "Authorization": "Bearer sk-1234", + }, +}); + +ws.on("open", function open() { + console.log("Connected — guardrails active:", guardrails); +}); + +ws.on("message", function incoming(message) { + const data = JSON.parse(message); + if (data.type === "error") { + // Guardrail block is sent as an error event before the connection closes + console.error("Guardrail error:", data.error.message); + } +}); + +ws.on("close", function close(code, reason) { + console.log("Closed:", code, reason.toString()); + // code 1011 = blocked by guardrail at pre_call +}); +``` + +Or with Python: + +```python +import asyncio +import websockets + +async def main(): + url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=your-guardrail-name" + async with websockets.connect( + url, + additional_headers={"Authorization": "Bearer sk-1234"}, + ) as ws: + print("Connected — guardrail active") + async for msg in ws: + import json + data = json.loads(msg) + if data["type"] == "error": + print("Guardrail blocked:", data["error"]["message"]) + break + +asyncio.run(main()) +``` + +When a guardrail blocks the request, the proxy sends an `error` event over the WebSocket and then closes the connection: + +```json +{ + "type": "error", + "error": { + "type": "guardrail_error", + "message": "Guardrail blocked this request: " + } +} +``` + +## Logging To prevent requests from being dropped, by default LiteLLM just logs these event types: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f0b1e66818..be76c2ac5f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -32,6 +32,8 @@ from typing import ( ) import anyio +import websockets +import websockets.exceptions from pydantic import BaseModel, Json from litellm._uuid import uuid @@ -392,7 +394,6 @@ from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router -from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router from litellm.proxy.management_endpoints.project_endpoints import ( router as project_router, ) @@ -418,6 +419,7 @@ from litellm.proxy.management_endpoints.ui_sso import ( get_disabled_non_admin_personal_key_creation, ) from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router +from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( router as user_agent_analytics_router, ) @@ -7358,6 +7360,10 @@ async def realtime_websocket_endpoint( intent: str = fastapi.Query( None, description="The intent of the websocket connection." ), + guardrails: Optional[str] = fastapi.Query( + None, + description="Comma-separated list of guardrail names to apply to this request.", + ), user_api_key_dict=Depends(user_api_key_auth_websocket), ): requested_protocols = [ @@ -7375,12 +7381,16 @@ async def realtime_websocket_endpoint( RealtimeQueryParams, dict(_realtime_query_params_template(model, intent)) ) - data = { + data: Dict[str, Any] = { "model": model, "websocket": websocket, "query_params": query_params, # Only explicit params } + # Pass guardrails into data so pre-call guardrail processing picks them up + if guardrails: + data["guardrails"] = [g.strip() for g in guardrails.split(",") if g.strip()] + # Use raw ASGI headers (already lowercase bytes) to avoid extra work headers_list = list(websocket.scope.get("headers") or []) @@ -7398,6 +7408,10 @@ async def realtime_websocket_endpoint( ### ROUTE THE REQUEST ### base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + + # Phase 1: pre-call processing (auth, guardrails, rate limits). + # Errors here (e.g. guardrail block) are sent back to the client as an + # error event before closing, so the caller knows what happened. try: ( data, @@ -7417,6 +7431,27 @@ async def realtime_websocket_endpoint( model=model, route_type="_arealtime", ) + except Exception as e: + verbose_proxy_logger.exception("Realtime pre-call error") + try: + await websocket.send_text( + json.dumps( + { + "type": "error", + "error": { + "type": "guardrail_error", + "message": str(e), + }, + } + ) + ) + except Exception: + pass + await websocket.close(code=1011, reason="Pre-call error") + return + + # Phase 2: route to upstream LLM. + try: data["user_api_key_dict"] = user_api_key_dict llm_call = await route_request( data=data, @@ -7424,7 +7459,6 @@ async def realtime_websocket_endpoint( llm_router=llm_router, user_model=user_model, ) - await llm_call except websockets.exceptions.InvalidStatusCode as e: # type: ignore verbose_proxy_logger.exception("Invalid status code") diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 3e165c7065..df04eece28 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -1832,6 +1832,7 @@ const ChatUI: React.FC = ({ accessToken={apiKeySource === "session" ? accessToken || "" : apiKey} selectedModel={selectedModel || ""} customProxyBaseUrl={customProxyBaseUrl || undefined} + selectedGuardrails={selectedGuardrails.length > 0 ? selectedGuardrails : undefined} /> ) : ( <> diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx index 87305385bc..a3dd864b89 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx @@ -24,12 +24,14 @@ interface RealtimePlaygroundProps { accessToken: string; selectedModel: string; customProxyBaseUrl?: string; + selectedGuardrails?: string[]; } const RealtimePlayground: React.FC = ({ accessToken, selectedModel, customProxyBaseUrl, + selectedGuardrails, }) => { const [messages, setMessages] = useState([]); const [inputText, setInputText] = useState(""); @@ -107,7 +109,10 @@ const RealtimePlayground: React.FC = ({ const baseUrl = customProxyBaseUrl || getProxyBaseUrl(); const wsBase = baseUrl.replace(/^http/, "ws"); - const url = `${wsBase}/v1/realtime?model=${encodeURIComponent(selectedModel)}`; + let url = `${wsBase}/v1/realtime?model=${encodeURIComponent(selectedModel)}`; + if (selectedGuardrails && selectedGuardrails.length > 0) { + url += `&guardrails=${encodeURIComponent(selectedGuardrails.join(","))}`; + } const ws = new WebSocket(url, ["realtime", `openai-insecure-api-key.${accessToken}`]); @@ -197,7 +202,7 @@ const RealtimePlayground: React.FC = ({ addMessage("status", `Connection failed: ${err.message}`); setIsConnecting(false); } - }, [accessToken, selectedModel, selectedVoice, customProxyBaseUrl, addMessage, appendAssistantText, playAudioChunk]); + }, [accessToken, selectedModel, selectedVoice, customProxyBaseUrl, selectedGuardrails, addMessage, appendAssistantText, playAudioChunk]); const disconnect = useCallback(() => { stopRecording();