diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 5d9b38cc1f..76d3acb7b1 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -95,7 +95,7 @@ print(response) - `router.image_generation()` - completion calls in OpenAI `/v1/images/generations` endpoint format - `router.aimage_generation()` - async image generation calls -### Advanced - Routing Strategies +## Advanced - Routing Strategies #### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based Router provides 4 strategies for routing your calls across multiple deployments: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f1d824b864..293d06023a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -720,6 +720,10 @@ class ConfigGeneralSettings(LiteLLMBase): None, description="List of alerting types. By default it is all alerts", ) + alert_to_webhook_url: Optional[Dict] = Field( + None, + description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`", + ) alerting_threshold: Optional[int] = Field( None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4f5b647725..87bf8df5c1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2632,9 +2632,17 @@ class ProxyConfig: if "alert_types" in _general_settings: general_settings["alert_types"] = _general_settings["alert_types"] proxy_logging_obj.alert_types = general_settings["alert_types"] - proxy_logging_obj.slack_alerting_instance.alert_types = general_settings[ - "alert_types" + proxy_logging_obj.slack_alerting_instance.update_values( + alert_types=general_settings["alert_types"] + ) + + if "alert_to_webhook_url" in _general_settings: + general_settings["alert_to_webhook_url"] = _general_settings[ + "alert_to_webhook_url" ] + proxy_logging_obj.slack_alerting_instance.update_values( + alert_to_webhook_url=general_settings["alert_to_webhook_url"] + ) # router settings if llm_router is not None and prisma_client is not None: @@ -8592,6 +8600,7 @@ async def get_config(): # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) + alerting_data = [] if "slack" in _alerting: _slack_vars = [ "SLACK_WEBHOOK_URL", @@ -8600,7 +8609,8 @@ async def get_config(): for _var in _slack_vars: env_variable = environment_variables.get(_var, None) if env_variable is None: - _slack_env_vars[_var] = None + _value = os.getenv("SLACK_WEBHOOK_URL", None) + _slack_env_vars[_var] = _value else: # decode + decrypt the value decoded_b64 = base64.b64decode(env_variable) @@ -8613,19 +8623,23 @@ async def get_config(): _all_alert_types = ( proxy_logging_obj.slack_alerting_instance._all_possible_alert_types() ) - _data_to_return.append( + _alerts_to_webhook = ( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + ) + alerting_data.append( { "name": "slack", "variables": _slack_env_vars, - "alerting_types": _alerting_types, - "all_alert_types": _all_alert_types, + "active_alerts": _alerting_types, + "alerts_to_webhook": _alerts_to_webhook, } ) _router_settings = llm_router.get_settings() return { "status": "success", - "data": _data_to_return, + "callbacks": _data_to_return, + "alerts": alerting_data, "router_settings": _router_settings, } except Exception as e: @@ -8742,10 +8756,51 @@ async def health_services_endpoint( } if "slack" in general_settings.get("alerting", []): - test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" - await proxy_logging_obj.alerting_handler( - message=test_message, level="Low", alert_type="budget_alerts" - ) + # test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" + # check if user has opted into unique_alert_webhooks + if ( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + is not None + ): + for ( + alert_type + ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url: + """ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + """ + # only test alert if it's in active alert types + if ( + proxy_logging_obj.slack_alerting_instance.alert_types + is not None + and alert_type + not in proxy_logging_obj.slack_alerting_instance.alert_types + ): + continue + test_message = "default test message" + if alert_type == "llm_exceptions": + test_message = f"LLM Exception test alert" + elif alert_type == "llm_too_slow": + test_message = f"LLM Too Slow test alert" + elif alert_type == "llm_requests_hanging": + test_message = f"LLM Requests Hanging test alert" + elif alert_type == "budget_alerts": + test_message = f"Budget Alert test alert" + elif alert_type == "db_exceptions": + test_message = f"DB Exception test alert" + + await proxy_logging_obj.alerting_handler( + message=test_message, level="Low", alert_type=alert_type + ) + else: + await proxy_logging_obj.alerting_handler( + message="This is a test slack alert message", + level="Low", + alert_type="budget_alerts", + ) return { "status": "success", "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel", @@ -8763,7 +8818,7 @@ async def health_services_endpoint( message=getattr(e, "detail", f"Authentication Error({str(e)})"), type="auth_error", param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): raise e @@ -8771,7 +8826,7 @@ async def health_services_endpoint( message="Authentication Error, " + str(e), type="auth_error", param=getattr(e, "param", "None"), - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 264964a940..db03a2bd00 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1274,7 +1274,7 @@ export const serviceHealthCheck= async (accessToken: String, service: String) => } const data = await response.json(); - message.success(`Test request to ${service} made - check logs on ${service} dashboard!`); + message.success(`Test request to ${service} made - check logs/alerts on ${service} to verify`); // You can add additional logic here based on the response if needed return data; } catch (error) { diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 8285f35b80..1d015386a1 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -15,7 +15,13 @@ import { Grid, Button, TextInput, + Switch, Col, + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; @@ -45,10 +51,29 @@ const Settings: React.FC = ({ userID, }) => { const [callbacks, setCallbacks] = useState([]); + const [alerts, setAlerts] = useState([]); const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); const [selectedAlertValues, setSelectedAlertValues] = useState([]); + const [catchAllWebhookURL, setCatchAllWebhookURL] = useState(""); + const [alertToWebhooks, setAlertToWebhooks] = useState([]); + const [activeAlerts, setActiveAlerts] = useState([]); + + + const handleSwitchChange = (alertName: string) => { + if (activeAlerts.includes(alertName)) { + setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName)); + } else { + setActiveAlerts([...activeAlerts, alertName]); + } + }; + const alerts_to_UI_NAME: Record = { + "llm_exceptions": "LLM Exceptions", + "llm_too_slow": "LLM Responses Too Slow", + "llm_requests_hanging": "LLM Requests Hanging", + "budget_alerts": "Budget Alerts (API Keys, Users)" + } useEffect(() => { if (!accessToken || !userRole || !userID) { @@ -56,11 +81,35 @@ const Settings: React.FC = ({ } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); - let callbacks_data = data.data; + let callbacks_data = data.callbacks; setCallbacks(callbacks_data); + + let alerts_data = data.alerts; + console.log("alerts_data", alerts_data); + if (alerts_data) { + if (alerts_data.length > 0) { + let _alert_info = alerts_data[0]; + console.log("_alert_info", _alert_info); + let catch_all_webhook = _alert_info.variables.SLACK_WEBHOOK_URL; + console.log("catch_all_webhook", catch_all_webhook); + + let active_alerts = _alert_info.active_alerts; + setActiveAlerts(active_alerts); + setCatchAllWebhookURL(catch_all_webhook); + setAlertToWebhooks(_alert_info.alerts_to_webhook); + + } + } + + setAlerts(alerts_data); }); }, [accessToken, userRole, userID]); + + const isAlertOn = (alertName: string) => { + return activeAlerts && activeAlerts.includes(alertName); + } + const handleAddCallback = () => { console.log("Add callback clicked"); setIsModalVisible(true); @@ -78,6 +127,40 @@ const Settings: React.FC = ({ console.log('Selected values:', values); }; + const handleSaveAlerts = () => { + if (!accessToken) { + return; + } + + const updatedAlertToWebhooks = {}; + Object.entries(alerts_to_UI_NAME).forEach(([key, value]) => { + const webhookInput = document.querySelector(`input[name="${key}"]`) as HTMLInputElement; + console.log("key", key); + console.log("webhookInput", webhookInput); + const newWebhookValue = webhookInput?.value || ''; + console.log("newWebhookValue", newWebhookValue); + updatedAlertToWebhooks[key] = newWebhookValue; + }); + + console.log("updatedAlertToWebhooks", updatedAlertToWebhooks); + + const payload = { + general_settings: { + alert_to_webhook_url: updatedAlertToWebhooks, + alert_types: activeAlerts + }, + }; + + console.log("payload", payload); + + try { + setCallbacksCall(accessToken, payload); + } catch (error) { + message.error('Failed to update alerts: ' + error, 20); + } + + message.success('Alerts updated successfully'); + }; const handleSaveChanges = (callback: any) => { if (!accessToken) { return; @@ -92,9 +175,6 @@ const Settings: React.FC = ({ const payload = { environment_variables: updatedVariables, - general_settings: { - alert_types: selectedAlertValues - } }; try { @@ -186,71 +266,114 @@ const Settings: React.FC = ({ return (
- Logging Callbacks - - - - - Callback - Callback Env Vars - - - - {callbacks.map((callback, index) => ( - - - {callback.name} - - -
    - {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( -
  • - {key} - {key === "LANGFUSE_HOST" ? ( -

    default value=https://cloud.langfuse.com

    - ) : ( -
    - )} - -
    • -))} -
- {callback.all_alert_types && ( -
- Alerting Types - -
- )} -
+ + + Callback + Callback Env Vars + + + + {callbacks.map((callback, index) => ( + + + {callback.name} + + +
    + {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( +
  • + {key} + {key === "LANGFUSE_HOST" ? ( +

    default value=https://cloud.langfuse.com

    + ) : ( +
    + )} + +
    • + ))} +
+ + + + + ))} + +
+ + + + + + + + + Alerts are only supported for Slack Webhook URLs. Get your webhook urls from here + + + + + + Slack Webhook URL + + + + + {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => ( + + + handleSwitchChange(key)} + /> + + + {value} + + + + + + + + ))} + +
+ - - - - ))} - - - - + + + + + + + - + + + + + + = ({ > @@ -297,18 +419,6 @@ const Settings: React.FC = ({ )} - {selectedCallback === 'slack' && ( - - - - )} -
Save