feat(realtime): guardrails support for /v1/realtime WebSocket endpoint (#22152)

* feat(realtime): add guardrails query param to /v1/realtime WebSocket endpoint - Add 'guardrails' query param (comma-separated) to realtime_websocket_endpoint - Import websockets and websockets.exceptions at module level (fixes NameError in except clause) - Split try/except into Phase 1 (pre-call) and Phase 2 (routing) so guardrail errors send back a typed error event before closing, while upstream errors close silently with 1011 * feat(ui): pass selectedGuardrails from sidebar to RealtimePlayground WebSocket URL * docs(realtime): add guardrails section with dynamic passing examples
2026-07-13 09:06:18 +00:00 · 2026-02-25 21:34:22 -08:00
parent 573969a703
commit 82cd14ea1d
4 changed files with 127 additions and 6 deletions
@@ -110,7 +110,88 @@ ws.on("error", function handleError(error) {
 });
 ```

-## Logging 
+## Guardrails
+
+You can apply [LiteLLM guardrails](https://docs.litellm.ai/docs/proxy/guardrails/quick_start) to realtime sessions.
+
+### Set guardrails on a key or team
+
+The easiest production setup — attach guardrails to a virtual key or team so they always apply automatically, without any client-side changes.
+
+See [Virtual Keys → Guardrails](https://docs.litellm.ai/docs/proxy/virtual_keys#guardrails) and [Teams → Guardrails](https://docs.litellm.ai/docs/proxy/team_budgets).
+
+### Pass guardrails dynamically (easy testing)
+
+Pass `guardrails` as a query param when opening the WebSocket.
+Useful for testing guardrails without modifying key/team config.
+
+```js
+// node test.js
+const WebSocket = require("ws");
+
+const guardrails = ["your-guardrail-name"]; // comma-separated list
+const url = `ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=${guardrails.join(",")}`;
+
+const ws = new WebSocket(url, {
+    headers: {
+        "Authorization": "Bearer sk-1234",
+    },
+});
+
+ws.on("open", function open() {
+    console.log("Connected — guardrails active:", guardrails);
+});
+
+ws.on("message", function incoming(message) {
+    const data = JSON.parse(message);
+    if (data.type === "error") {
+        // Guardrail block is sent as an error event before the connection closes
+        console.error("Guardrail error:", data.error.message);
+    }
+});
+
+ws.on("close", function close(code, reason) {
+    console.log("Closed:", code, reason.toString());
+    // code 1011 = blocked by guardrail at pre_call
+});
+```
+
+Or with Python:
+
+```python
+import asyncio
+import websockets
+
+async def main():
+    url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=your-guardrail-name"
+    async with websockets.connect(
+        url,
+        additional_headers={"Authorization": "Bearer sk-1234"},
+    ) as ws:
+        print("Connected — guardrail active")
+        async for msg in ws:
+            import json
+            data = json.loads(msg)
+            if data["type"] == "error":
+                print("Guardrail blocked:", data["error"]["message"])
+                break
+
+asyncio.run(main())
+```
+
+When a guardrail blocks the request, the proxy sends an `error` event over the WebSocket and then closes the connection:
+
+```json
+{
+    "type": "error",
+    "error": {
+        "type": "guardrail_error",
+        "message": "Guardrail blocked this request: <reason>"
+    }
+}
+```
+
+## Logging

 To prevent requests from being dropped, by default LiteLLM just logs these event types:

@@ -32,6 +32,8 @@ from typing import (
 )

 import anyio
+import websockets
+import websockets.exceptions
 from pydantic import BaseModel, Json

 from litellm._uuid import uuid
@@ -392,7 +394,6 @@ from litellm.proxy.management_endpoints.organization_endpoints import (
    router as organization_router,
 )
 from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router
-from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
 from litellm.proxy.management_endpoints.project_endpoints import (
    router as project_router,
 )
@@ -418,6 +419,7 @@ from litellm.proxy.management_endpoints.ui_sso import (
    get_disabled_non_admin_personal_key_creation,
 )
 from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
+from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
 from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
    router as user_agent_analytics_router,
 )
@@ -7358,6 +7360,10 @@ async def realtime_websocket_endpoint(
    intent: str = fastapi.Query(
        None, description="The intent of the websocket connection."
    ),
+    guardrails: Optional[str] = fastapi.Query(
+        None,
+        description="Comma-separated list of guardrail names to apply to this request.",
+    ),
    user_api_key_dict=Depends(user_api_key_auth_websocket),
 ):
    requested_protocols = [
@@ -7375,12 +7381,16 @@ async def realtime_websocket_endpoint(
        RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))
    )

-    data = {
+    data: Dict[str, Any] = {
        "model": model,
        "websocket": websocket,
        "query_params": query_params,  # Only explicit params
    }

+    # Pass guardrails into data so pre-call guardrail processing picks them up
+    if guardrails:
+        data["guardrails"] = [g.strip() for g in guardrails.split(",") if g.strip()]
+
    # Use raw ASGI headers (already lowercase bytes) to avoid extra work
    headers_list = list(websocket.scope.get("headers") or [])

@@ -7398,6 +7408,10 @@ async def realtime_websocket_endpoint(

    ### ROUTE THE REQUEST ###
    base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
+
+    # Phase 1: pre-call processing (auth, guardrails, rate limits).
+    # Errors here (e.g. guardrail block) are sent back to the client as an
+    # error event before closing, so the caller knows what happened.
    try:
        (
            data,
@@ -7417,6 +7431,27 @@ async def realtime_websocket_endpoint(
            model=model,
            route_type="_arealtime",
        )
+    except Exception as e:
+        verbose_proxy_logger.exception("Realtime pre-call error")
+        try:
+            await websocket.send_text(
+                json.dumps(
+                    {
+                        "type": "error",
+                        "error": {
+                            "type": "guardrail_error",
+                            "message": str(e),
+                        },
+                    }
+                )
+            )
+        except Exception:
+            pass
+        await websocket.close(code=1011, reason="Pre-call error")
+        return
+
+    # Phase 2: route to upstream LLM.
+    try:
        data["user_api_key_dict"] = user_api_key_dict
        llm_call = await route_request(
            data=data,
@@ -7424,7 +7459,6 @@ async def realtime_websocket_endpoint(
            llm_router=llm_router,
            user_model=user_model,
        )
-
        await llm_call
    except websockets.exceptions.InvalidStatusCode as e:  # type: ignore
        verbose_proxy_logger.exception("Invalid status code")
@@ -1832,6 +1832,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
                accessToken={apiKeySource === "session" ? accessToken || "" : apiKey}
                selectedModel={selectedModel || ""}
                customProxyBaseUrl={customProxyBaseUrl || undefined}
+                selectedGuardrails={selectedGuardrails.length > 0 ? selectedGuardrails : undefined}
              />
            ) : (
            <>
@@ -24,12 +24,14 @@ interface RealtimePlaygroundProps {
  accessToken: string;
  selectedModel: string;
  customProxyBaseUrl?: string;
+  selectedGuardrails?: string[];
 }

 const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
  accessToken,
  selectedModel,
  customProxyBaseUrl,
+  selectedGuardrails,
 }) => {
  const [messages, setMessages] = useState<RealtimeMessage[]>([]);
  const [inputText, setInputText] = useState("");
@@ -107,7 +109,10 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({

      const baseUrl = customProxyBaseUrl || getProxyBaseUrl();
      const wsBase = baseUrl.replace(/^http/, "ws");
-      const url = `${wsBase}/v1/realtime?model=${encodeURIComponent(selectedModel)}`;
+      let url = `${wsBase}/v1/realtime?model=${encodeURIComponent(selectedModel)}`;
+      if (selectedGuardrails && selectedGuardrails.length > 0) {
+        url += `&guardrails=${encodeURIComponent(selectedGuardrails.join(","))}`;
+      }

      const ws = new WebSocket(url, ["realtime", `openai-insecure-api-key.${accessToken}`]);

@@ -197,7 +202,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
      addMessage("status", `Connection failed: ${err.message}`);
      setIsConnecting(false);
    }
-  }, [accessToken, selectedModel, selectedVoice, customProxyBaseUrl, addMessage, appendAssistantText, playAudioChunk]);
+  }, [accessToken, selectedModel, selectedVoice, customProxyBaseUrl, selectedGuardrails, addMessage, appendAssistantText, playAudioChunk]);

  const disconnect = useCallback(() => {
    stopRecording();