Merge pull request #3309 from BerriAI/litellm_set_unique_webhooks_per_alert

[FEAT - UI + Backend] Proxy - set unique webhooks per alert

docs/my-website/docs/routing.md

@@ -95,7 +95,7 @@ print(response)
 - `router.image_generation()` - completion calls in OpenAI `/v1/images/generations` endpoint format
 - `router.aimage_generation()` - async image generation calls
 
-### Advanced - Routing Strategies
+## Advanced - Routing Strategies
 #### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based
 
 Router provides 4 strategies for routing your calls across multiple deployments: 

litellm/proxy/_types.py

@@ -720,6 +720,10 @@ class ConfigGeneralSettings(LiteLLMBase):
         None,
         description="List of alerting types. By default it is all alerts",
     )
+    alert_to_webhook_url: Optional[Dict] = Field(
+        None,
+        description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`",
+    )
 
     alerting_threshold: Optional[int] = Field(
         None,

litellm/proxy/proxy_server.py

@@ -2632,9 +2632,17 @@ class ProxyConfig:
         if "alert_types" in _general_settings:
             general_settings["alert_types"] = _general_settings["alert_types"]
             proxy_logging_obj.alert_types = general_settings["alert_types"]
-            proxy_logging_obj.slack_alerting_instance.alert_types = general_settings[
-                "alert_types"
+            proxy_logging_obj.slack_alerting_instance.update_values(
+                alert_types=general_settings["alert_types"]
+            )
+
+        if "alert_to_webhook_url" in _general_settings:
+            general_settings["alert_to_webhook_url"] = _general_settings[
+                "alert_to_webhook_url"
             ]
+            proxy_logging_obj.slack_alerting_instance.update_values(
+                alert_to_webhook_url=general_settings["alert_to_webhook_url"]
+            )
 
         # router settings
         if llm_router is not None and prisma_client is not None:
@@ -8592,6 +8600,7 @@ async def get_config():
 
         # Check if slack alerting is on
         _alerting = _general_settings.get("alerting", [])
+        alerting_data = []
         if "slack" in _alerting:
             _slack_vars = [
                 "SLACK_WEBHOOK_URL",
@@ -8600,7 +8609,8 @@ async def get_config():
             for _var in _slack_vars:
                 env_variable = environment_variables.get(_var, None)
                 if env_variable is None:
-                    _slack_env_vars[_var] = None
+                    _value = os.getenv("SLACK_WEBHOOK_URL", None)
+                    _slack_env_vars[_var] = _value
                 else:
                     # decode + decrypt the value
                     decoded_b64 = base64.b64decode(env_variable)
@@ -8613,19 +8623,23 @@ async def get_config():
             _all_alert_types = (
                 proxy_logging_obj.slack_alerting_instance._all_possible_alert_types()
             )
-            _data_to_return.append(
+            _alerts_to_webhook = (
+                proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
+            )
+            alerting_data.append(
                 {
                     "name": "slack",
                     "variables": _slack_env_vars,
-                    "alerting_types": _alerting_types,
-                    "all_alert_types": _all_alert_types,
+                    "active_alerts": _alerting_types,
+                    "alerts_to_webhook": _alerts_to_webhook,
                 }
             )
 
         _router_settings = llm_router.get_settings()
         return {
             "status": "success",
-            "data": _data_to_return,
+            "callbacks": _data_to_return,
+            "alerts": alerting_data,
             "router_settings": _router_settings,
         }
     except Exception as e:
@@ -8742,10 +8756,51 @@ async def health_services_endpoint(
             }
 
         if "slack" in general_settings.get("alerting", []):
-            test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
-            await proxy_logging_obj.alerting_handler(
-                message=test_message, level="Low", alert_type="budget_alerts"
-            )
+            # test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
+            # check if user has opted into unique_alert_webhooks
+            if (
+                proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
+                is not None
+            ):
+                for (
+                    alert_type
+                ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url:
+                    """
+                    "llm_exceptions",
+                    "llm_too_slow",
+                    "llm_requests_hanging",
+                    "budget_alerts",
+                    "db_exceptions",
+                    """
+                    # only test alert if it's in active alert types
+                    if (
+                        proxy_logging_obj.slack_alerting_instance.alert_types
+                        is not None
+                        and alert_type
+                        not in proxy_logging_obj.slack_alerting_instance.alert_types
+                    ):
+                        continue
+                    test_message = "default test message"
+                    if alert_type == "llm_exceptions":
+                        test_message = f"LLM Exception test alert"
+                    elif alert_type == "llm_too_slow":
+                        test_message = f"LLM Too Slow test alert"
+                    elif alert_type == "llm_requests_hanging":
+                        test_message = f"LLM Requests Hanging test alert"
+                    elif alert_type == "budget_alerts":
+                        test_message = f"Budget Alert test alert"
+                    elif alert_type == "db_exceptions":
+                        test_message = f"DB Exception test alert"
+
+                    await proxy_logging_obj.alerting_handler(
+                        message=test_message, level="Low", alert_type=alert_type
+                    )
+            else:
+                await proxy_logging_obj.alerting_handler(
+                    message="This is a test slack alert message",
+                    level="Low",
+                    alert_type="budget_alerts",
+                )
             return {
                 "status": "success",
                 "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
@@ -8763,7 +8818,7 @@ async def health_services_endpoint(
                 message=getattr(e, "detail", f"Authentication Error({str(e)})"),
                 type="auth_error",
                 param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
             )
         elif isinstance(e, ProxyException):
             raise e
@@ -8771,7 +8826,7 @@ async def health_services_endpoint(
             message="Authentication Error, " + str(e),
             type="auth_error",
             param=getattr(e, "param", "None"),
-            code=status.HTTP_401_UNAUTHORIZED,
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
 
 

ui/litellm-dashboard/src/components/networking.tsx

@@ -1274,7 +1274,7 @@ export const serviceHealthCheck= async (accessToken: String, service: String) =>
     }
     
     const data = await response.json();
-    message.success(`Test request to ${service} made - check logs on ${service} dashboard!`);
+    message.success(`Test request to ${service} made - check logs/alerts on ${service} to verify`);
     // You can add additional logic here based on the response if needed
     return data;
   } catch (error) {

ui/litellm-dashboard/src/components/settings.tsx

@@ -15,7 +15,13 @@ import {
   Grid,
   Button,
   TextInput,
+  Switch,
   Col,
+  TabPanel, 
+  TabPanels, 
+  TabGroup, 
+  TabList, 
+  Tab
 } from "@tremor/react";
 import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
 import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
@@ -45,10 +51,29 @@ const Settings: React.FC<SettingsPageProps> = ({
   userID,
 }) => {
   const [callbacks, setCallbacks] = useState<any[]>([]);
+  const [alerts, setAlerts] = useState<any[]>([]);
   const [isModalVisible, setIsModalVisible] = useState(false);
   const [form] = Form.useForm();
   const [selectedCallback, setSelectedCallback] = useState<string | null>(null);
   const [selectedAlertValues, setSelectedAlertValues] = useState([]);
+  const [catchAllWebhookURL, setCatchAllWebhookURL] = useState<string>("");
+  const [alertToWebhooks, setAlertToWebhooks] = useState<any[]>([]);
+  const [activeAlerts, setActiveAlerts] = useState<string[]>([]);
+
+
+  const handleSwitchChange = (alertName: string) => {
+    if (activeAlerts.includes(alertName)) {
+      setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName));
+    } else {
+      setActiveAlerts([...activeAlerts, alertName]);
+    }
+  };
+  const alerts_to_UI_NAME: Record<string, string> = {
+    "llm_exceptions": "LLM Exceptions",
+    "llm_too_slow": "LLM Responses Too Slow",
+    "llm_requests_hanging": "LLM Requests Hanging",
+    "budget_alerts": "Budget Alerts (API Keys, Users)"
+  }
 
   useEffect(() => {
     if (!accessToken || !userRole || !userID) {
@@ -56,11 +81,35 @@ const Settings: React.FC<SettingsPageProps> = ({
     }
     getCallbacksCall(accessToken, userID, userRole).then((data) => {
       console.log("callbacks", data);
-      let callbacks_data = data.data;
+      let callbacks_data = data.callbacks;
       setCallbacks(callbacks_data);
+
+      let alerts_data = data.alerts;
+      console.log("alerts_data", alerts_data);
+      if (alerts_data) {
+        if (alerts_data.length > 0) {
+          let _alert_info = alerts_data[0];
+          console.log("_alert_info", _alert_info);
+          let catch_all_webhook = _alert_info.variables.SLACK_WEBHOOK_URL;
+          console.log("catch_all_webhook", catch_all_webhook);
+
+          let active_alerts = _alert_info.active_alerts;
+          setActiveAlerts(active_alerts);
+          setCatchAllWebhookURL(catch_all_webhook);
+          setAlertToWebhooks(_alert_info.alerts_to_webhook);
+
+        }
+      }
+
+      setAlerts(alerts_data);
     });
   }, [accessToken, userRole, userID]);
 
+
+  const isAlertOn = (alertName: string) => {
+    return activeAlerts && activeAlerts.includes(alertName);
+  }
+
   const handleAddCallback = () => {
     console.log("Add callback clicked");
     setIsModalVisible(true);
@@ -78,6 +127,40 @@ const Settings: React.FC<SettingsPageProps> = ({
     console.log('Selected values:', values);
   };
 
+  const handleSaveAlerts = () => {
+    if (!accessToken) {
+      return;
+    }
+  
+    const updatedAlertToWebhooks = {};
+    Object.entries(alerts_to_UI_NAME).forEach(([key, value]) => {
+      const webhookInput = document.querySelector(`input[name="${key}"]`) as HTMLInputElement;
+      console.log("key", key);
+      console.log("webhookInput", webhookInput);
+      const newWebhookValue = webhookInput?.value || '';
+      console.log("newWebhookValue", newWebhookValue);
+      updatedAlertToWebhooks[key] = newWebhookValue;
+    });
+
+    console.log("updatedAlertToWebhooks", updatedAlertToWebhooks);
+
+    const payload = {
+      general_settings: {
+        alert_to_webhook_url: updatedAlertToWebhooks,
+        alert_types: activeAlerts
+      },
+    };
+
+    console.log("payload", payload);
+  
+    try {
+      setCallbacksCall(accessToken, payload);
+    } catch (error) {
+      message.error('Failed to update alerts: ' + error, 20);
+    }
+  
+    message.success('Alerts updated successfully');
+  };
   const handleSaveChanges = (callback: any) => {
     if (!accessToken) {
       return;
@@ -92,9 +175,6 @@ const Settings: React.FC<SettingsPageProps> = ({
 
     const payload = {
       environment_variables: updatedVariables,
-      general_settings: {
-        alert_types: selectedAlertValues
-      }
     };
 
     try {
@@ -186,71 +266,114 @@ const Settings: React.FC<SettingsPageProps> = ({
   return (
     <div className="w-full mx-4">
       <Grid numItems={1} className="gap-2 p-8 w-full mt-2">
-      <Title>Logging Callbacks</Title>
-        <Card >
-          <Table>
-            <TableHead>
-              <TableRow>
-                <TableHeaderCell>Callback</TableHeaderCell>
-                <TableHeaderCell>Callback Env Vars</TableHeaderCell>
-              </TableRow>
-            </TableHead>
-            <TableBody>
-  {callbacks.map((callback, index) => (
-    <TableRow key={index}>
-      <TableCell>
-        <Badge color="emerald">{callback.name}</Badge>
-      </TableCell>
-      <TableCell>
-        <ul>
-        {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => (
-  <li key={key}>
-    <Text className="mt-2">{key}</Text>
-    {key === "LANGFUSE_HOST" ? (
-      <p>default value=https://cloud.langfuse.com</p>
-    ) : (
-      <div></div>
-    )}
-    <TextInput name={key} defaultValue={value as string} type="password" />
-  </li>
-))}
-        </ul>
-        {callback.all_alert_types && (
-          <div>
-            <Text className="mt-2">Alerting Types</Text>
-            <Select
-              mode="multiple"
-              style={{ width: '100%' }}
-              placeholder="Select Alerting Types"
-              optionLabelProp="label"
-              onChange={handleChange}
-              defaultValue={callback.alerting_types}
-            >
-              {callback.all_alert_types.map((type: string) => (
-                <Select.Option key={type} value={type} label={type}>
-                  {type}
-                </Select.Option>
-              ))}
-            </Select>
-          </div>
-        )}
-        <Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
+        <TabGroup>
+        <TabList variant="line" defaultValue="1">
+          <Tab value="1">Logging Callbacks</Tab>
+          <Tab value="2">Alerting</Tab>
+        </TabList>
+        <TabPanels>
+          <TabPanel>
+          <Card >
+            <Table>
+              <TableHead>
+                <TableRow>
+                  <TableHeaderCell>Callback</TableHeaderCell>
+                  <TableHeaderCell>Callback Env Vars</TableHeaderCell>
+                </TableRow>
+              </TableHead>
+              <TableBody>
+    {callbacks.map((callback, index) => (
+      <TableRow key={index}>
+        <TableCell>
+          <Badge color="emerald">{callback.name}</Badge>
+        </TableCell>
+        <TableCell>
+          <ul>
+          {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => (
+    <li key={key}>
+      <Text className="mt-2">{key}</Text>
+      {key === "LANGFUSE_HOST" ? (
+        <p>default value=https://cloud.langfuse.com</p>
+      ) : (
+        <div></div>
+      )}
+      <TextInput name={key} defaultValue={value as string} type="password" />
+    </li>
+  ))}
+          </ul>
+          <Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
+            Save Changes
+          </Button>
+          <Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
+            Test Callback
+          </Button>
+        </TableCell>
+      </TableRow>
+    ))}
+  </TableBody>
+            </Table>
+              <Button size="xs" className="mt-2" onClick={handleAddCallback}>
+                Add Callback
+              </Button>
+              
+          </Card>
+          </TabPanel>
+
+          <TabPanel>
+
+          <Card>
+            <Text className="my-2">Alerts are only supported for Slack Webhook URLs. Get your webhook urls from <a href="https://api.slack.com/messaging/webhooks" target="_blank" style={{color: 'blue'}}>here</a></Text>
+        <Table>
+        <TableHead>
+          <TableRow>
+            <TableHeaderCell></TableHeaderCell>
+            <TableHeaderCell></TableHeaderCell>
+            <TableHeaderCell>Slack Webhook URL</TableHeaderCell>
+          </TableRow>
+        </TableHead>
+          
+        <TableBody>
+          {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => (
+            <TableRow key={index}>
+              <TableCell>
+              <Switch
+                    id="switch"
+                    name="switch"
+                    checked={isAlertOn(key)}
+                    onChange={() => handleSwitchChange(key)}
+                  />
+              </TableCell>
+              <TableCell>
+              <Text>{value}</Text>
+              </TableCell>
+              <TableCell>
+              <TextInput name={key} type="password" defaultValue={alertToWebhooks && alertToWebhooks[key] ? alertToWebhooks[key] : catchAllWebhookURL as string}>
+                
+              </TextInput>
+              </TableCell>
+            </TableRow>
+          ))}
+        </TableBody>
+        </Table>
+        <Button size="xs" className="mt-2" onClick={handleSaveAlerts}>
           Save Changes
         </Button>
-        <Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
-          Test Callback
-        </Button>
-      </TableCell>
-    </TableRow>
-  ))}
-</TableBody>
-          </Table>
-            <Button size="xs" className="mt-2" onClick={handleAddCallback}>
-              Add Callback
-            </Button>
-            
+
+        <Button onClick={() => serviceHealthCheck(accessToken, "slack")} className="mx-2">
+            Test Alerts
+          </Button>
+
+      
+
+
+
         </Card>
-        
+            
+          </TabPanel>
+          </TabPanels>
+          </TabGroup>
+
+      
       </Grid>
 
       <Modal
@@ -269,7 +392,6 @@ const Settings: React.FC<SettingsPageProps> = ({
           >
             <Select onChange={handleCallbackChange}>
               <Select.Option value="langfuse">langfuse</Select.Option>
-              <Select.Option value="slack">slack alerting</Select.Option>
             </Select>
           </Form.Item>
 
@@ -297,18 +419,6 @@ const Settings: React.FC<SettingsPageProps> = ({
             </>
           )}
 
-          {selectedCallback === 'slack' && (
-            <Form.Item
-              label="SLACK_WEBHOOK_URL"
-              name="slackWebhookUrl"
-              rules={[
-                { required: true, message: "Please enter the Slack webhook URL" },
-              ]}
-            >
-              <TextInput/>
-            </Form.Item>
-          )}
-
           <div style={{ textAlign: "right", marginTop: "10px" }}>
             <Button2 htmlType="submit">Save</Button2>
           </div>