diff --git a/docs/my-website/docs/proxy/pass_through_guardrails.md b/docs/my-website/docs/proxy/pass_through_guardrails.md index 272285e61c..cc3d36c866 100644 --- a/docs/my-website/docs/proxy/pass_through_guardrails.md +++ b/docs/my-website/docs/proxy/pass_through_guardrails.md @@ -1,5 +1,7 @@ # Guardrails on Pass-Through Endpoints +import Image from '@theme/IdealImage'; + ## Overview | Property | Details | @@ -10,7 +12,41 @@ ## Quick Start -### 1. Define guardrails and pass-through endpoint +You can configure guardrails on pass-through endpoints either via the **UI** (recommended) or **config file**. + +### Using the UI + +#### 1. Navigate to Pass-Through Endpoints + +Go to **Models + Endpoints** → Click **+ Add Pass-Through Endpoint** + +Add guardrails to pass-through endpoint + +Scroll to the **Guardrails** section and select which guardrails to enforce. + +:::tip Default Behavior +By default, you don't need to specify fields - LiteLLM will JSON dump the entire request/response payload and send it to the guardrail. +::: + +#### 2. Target Specific Fields (Optional) + +Configure field-level targeting + +To check only specific fields instead of the entire payload: + +1. Select your guardrails +2. In **Field Targeting (Optional)**, specify fields for each guardrail +3. Use the quick-add buttons (`+ query`, `+ documents[*]`) or type custom JSONPath expressions +4. **Request Fields (pre_call)**: Fields to check before sending to target API +5. **Response Fields (post_call)**: Fields to check in the response from target API + +**Example**: In the screenshot above, we set `query` as a request field, so only the `query` field is sent to the guardrail instead of the entire request. + +--- + +### Using Config File + +#### 1. Define guardrails and pass-through endpoint ```yaml showLineNumbers title="config.yaml" guardrails: @@ -31,13 +67,13 @@ general_settings: pii-guard: ``` -### 2. Start proxy +#### 2. Start proxy ```bash litellm --config config.yaml ``` -### 3. Test request +#### 3. Test request ```bash curl -X POST "http://localhost:4000/v1/rerank" \ diff --git a/docs/my-website/img/pt_guard1.png b/docs/my-website/img/pt_guard1.png new file mode 100644 index 0000000000..85b094a14b Binary files /dev/null and b/docs/my-website/img/pt_guard1.png differ diff --git a/docs/my-website/img/pt_guard2.png b/docs/my-website/img/pt_guard2.png new file mode 100644 index 0000000000..32481109bc Binary files /dev/null and b/docs/my-website/img/pt_guard2.png differ diff --git a/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py b/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py index cec20f3e04..c37703b9df 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py @@ -261,7 +261,7 @@ class PassthroughGuardrailHandler: for guardrail_name in normalized_config.keys(): guardrails_to_run[guardrail_name] = True verbose_proxy_logger.debug( - "Added passthrough-specific guardrail" + "Added passthrough-specific guardrail: %s", guardrail_name ) # Add org/team/key level guardrails using shared helper @@ -279,12 +279,12 @@ class PassthroughGuardrailHandler: if guardrail_name not in guardrails_to_run: guardrails_to_run[guardrail_name] = True verbose_proxy_logger.debug( - "Added inherited guardrail (key/team level)" + "Added inherited guardrail (key/team level): %s", guardrail_name ) verbose_proxy_logger.debug( - "Collected total guardrails for passthrough endpoint: %d", - len(guardrails_to_run), + "Collected guardrails for passthrough endpoint: %s", + list(guardrails_to_run.keys()), ) return guardrails_to_run if guardrails_to_run else None diff --git a/ui/litellm-dashboard/src/components/add_pass_through.tsx b/ui/litellm-dashboard/src/components/add_pass_through.tsx index 2332b9735d..ff724e42eb 100644 --- a/ui/litellm-dashboard/src/components/add_pass_through.tsx +++ b/ui/litellm-dashboard/src/components/add_pass_through.tsx @@ -27,6 +27,7 @@ import { passThroughItem } from "./pass_through_settings"; import RoutePreview from "./route_preview"; import NotificationsManager from "./molecules/notifications_manager"; import PassThroughSecuritySection from "./common_components/PassThroughSecuritySection"; +import PassThroughGuardrailsSection from "./common_components/PassThroughGuardrailsSection"; const { Option } = Select2; interface AddFallbacksProps { @@ -51,11 +52,13 @@ const AddPassThroughEndpoint: React.FC = ({ const [targetValue, setTargetValue] = useState(""); const [includeSubpath, setIncludeSubpath] = useState(true); const [authEnabled, setAuthEnabled] = useState(false); + const [guardrails, setGuardrails] = useState>({}); const handleCancel = () => { form.resetFields(); setPathValue(""); setTargetValue(""); setIncludeSubpath(true); + setGuardrails({}); setIsModalVisible(false); }; @@ -77,6 +80,12 @@ const AddPassThroughEndpoint: React.FC = ({ if (!premiumUser && 'auth' in formValues) { delete formValues.auth; } + + // Add guardrails to formValues (only if not empty) + if (guardrails && Object.keys(guardrails).length > 0) { + formValues.guardrails = guardrails; + } + console.log(`formValues: ${JSON.stringify(formValues)}`); const response = await createPassThroughEndpoint(accessToken, formValues); @@ -92,6 +101,7 @@ const AddPassThroughEndpoint: React.FC = ({ setPathValue(""); setTargetValue(""); setIncludeSubpath(true); + setGuardrails({}); setIsModalVisible(false); } catch (error) { NotificationsManager.fromBackend("Error creating pass-through endpoint: " + error); @@ -249,6 +259,14 @@ const AddPassThroughEndpoint: React.FC = ({ form.setFieldsValue({ auth: checked }); }} /> + + {/* Guardrails Section */} + + {/* Billing Section */} Billing diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx new file mode 100644 index 0000000000..fc6f783df0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx @@ -0,0 +1,246 @@ +import React, { useState, useEffect } from "react"; +import { Card, Title, Subtitle } from "@tremor/react"; +import { Form, Input, Select, Tooltip, Alert } from "antd"; +import { InfoCircleOutlined, PlusOutlined, DeleteOutlined } from "@ant-design/icons"; +import GuardrailSelector from "../guardrails/GuardrailSelector"; + +interface PassThroughGuardrailsSectionProps { + accessToken: string; + value?: Record; + onChange?: (guardrails: Record) => void; + disabled?: boolean; +} + +const PassThroughGuardrailsSection: React.FC = ({ + accessToken, + value = {}, + onChange, + disabled = false, +}) => { + const [selectedGuardrails, setSelectedGuardrails] = useState(Object.keys(value)); + const [guardrailSettings, setGuardrailSettings] = useState< + Record + >(value); + + // Sync external value changes + useEffect(() => { + setGuardrailSettings(value); + setSelectedGuardrails(Object.keys(value)); + }, [value]); + + const handleGuardrailChange = (guardrails: string[]) => { + setSelectedGuardrails(guardrails); + + // Create new settings object with selected guardrails + const newSettings: Record = {}; + guardrails.forEach((name) => { + // Preserve existing settings or set to null (uses entire payload) + newSettings[name] = guardrailSettings[name] || null; + }); + + setGuardrailSettings(newSettings); + if (onChange) { + onChange(newSettings); + } + }; + + const handleFieldChange = ( + guardrailName: string, + fieldType: "request_fields" | "response_fields", + fields: string[] + ) => { + const currentSettings = guardrailSettings[guardrailName] || {}; + const newSettings = { + ...guardrailSettings, + [guardrailName]: { + ...currentSettings, + [fieldType]: fields.length > 0 ? fields : undefined, + }, + }; + + // If no fields are set, set to null (entire payload) + if (!newSettings[guardrailName]?.request_fields && !newSettings[guardrailName]?.response_fields) { + newSettings[guardrailName] = null; + } + + setGuardrailSettings(newSettings); + if (onChange) { + onChange(newSettings); + } + }; + + return ( + + Guardrails + + Configure guardrails to enforce policies on requests and responses. Guardrails are opt-in for passthrough + endpoints. + + + + Field-Level Targeting{" "} + + (Learn More) + + + } + description={ +
+
+ Optionally specify which fields to check. If left empty, the entire request/response is sent to the guardrail. +
+
+
Common Examples:
+
query - Single field
+
documents[*].text - All text in documents array
+
messages[*].content - All message contents
+
+
+ } + type="info" + showIcon + className="mb-4" + /> + + + Select Guardrails + + + + + } + > + + + + {selectedGuardrails.length > 0 && ( +
+
+
Field Targeting (Optional)
+
+ 💡 Tip: Leave empty to check entire payload +
+
+ {selectedGuardrails.map((guardrailName) => ( + +
{guardrailName}
+
+
+
+
+ }> + + + +
+ + +
+
+ handleFieldChange(guardrailName, "response_fields", fields)} + disabled={disabled} + tokenSeparators={[","]} + /> +
+
+
+ ))} + + )} +
+ ); +}; + +export default PassThroughGuardrailsSection; + diff --git a/ui/litellm-dashboard/src/components/pass_through_info.tsx b/ui/litellm-dashboard/src/components/pass_through_info.tsx index c42a710caf..fe3b204b50 100644 --- a/ui/litellm-dashboard/src/components/pass_through_info.tsx +++ b/ui/litellm-dashboard/src/components/pass_through_info.tsx @@ -19,6 +19,7 @@ import { Eye, EyeOff } from "lucide-react"; import RoutePreview from "./route_preview"; import NotificationsManager from "./molecules/notifications_manager"; import PassThroughSecuritySection from "./common_components/PassThroughSecuritySection"; +import PassThroughGuardrailsSection from "./common_components/PassThroughGuardrailsSection"; export interface PassThroughInfoProps { endpointData: PassThroughEndpoint; @@ -37,6 +38,7 @@ interface PassThroughEndpoint { include_subpath?: boolean; cost_per_request?: number; auth?: boolean; + guardrails?: Record; } // Password field component for headers @@ -68,6 +70,9 @@ const PassThroughInfoView: React.FC = ({ const [loading, setLoading] = useState(false); const [isEditing, setIsEditing] = useState(false); const [authEnabled, setAuthEnabled] = useState(initialEndpointData?.auth || false); + const [guardrails, setGuardrails] = useState>( + initialEndpointData?.guardrails || {} + ); const [form] = Form.useForm(); const handleEndpointUpdate = async (values: any) => { @@ -92,6 +97,7 @@ const PassThroughInfoView: React.FC = ({ include_subpath: values.include_subpath, cost_per_request: values.cost_per_request, auth: premiumUser ? values.auth : undefined, + guardrails: guardrails && Object.keys(guardrails).length > 0 ? guardrails : undefined, }; await updatePassThroughEndpoint(accessToken, endpointData.id, updateData); @@ -214,6 +220,33 @@ const PassThroughInfoView: React.FC = ({ )} + + {endpointData.guardrails && Object.keys(endpointData.guardrails).length > 0 && ( + +
+ Guardrails + {Object.keys(endpointData.guardrails).length} guardrails configured +
+
+ {Object.entries(endpointData.guardrails).map(([name, settings]) => ( +
+
{name}
+ {settings && (settings.request_fields || settings.response_fields) && ( +
+ {settings.request_fields && ( +
Request fields: {settings.request_fields.join(", ")}
+ )} + {settings.response_fields && ( +
Response fields: {settings.response_fields.join(", ")}
+ )} +
+ )} + {!settings &&
Uses entire payload
} +
+ ))} +
+
+ )} {/* Settings Panel (only for admins) */} @@ -279,6 +312,14 @@ const PassThroughInfoView: React.FC = ({ }} /> +
+ +
+
Save Changes diff --git a/ui/litellm-dashboard/src/components/pass_through_settings.tsx b/ui/litellm-dashboard/src/components/pass_through_settings.tsx index 6e22533ed1..b37d3df7e4 100644 --- a/ui/litellm-dashboard/src/components/pass_through_settings.tsx +++ b/ui/litellm-dashboard/src/components/pass_through_settings.tsx @@ -39,6 +39,7 @@ export interface passThroughItem { include_subpath?: boolean; cost_per_request?: number; auth?: boolean; + guardrails?: Record; } // Password field component for headers