From fa83e2da0607f5d7dc3b613e46dba4324431510e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 13:43:51 -0700 Subject: [PATCH 01/11] docs - fix routing --- docs/my-website/docs/routing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 5d9b38cc1f..76d3acb7b1 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -95,7 +95,7 @@ print(response) - `router.image_generation()` - completion calls in OpenAI `/v1/images/generations` endpoint format - `router.aimage_generation()` - async image generation calls -### Advanced - Routing Strategies +## Advanced - Routing Strategies #### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based Router provides 4 strategies for routing your calls across multiple deployments: From 61f48aba6f4ee46e3abefe40b788267c5a700fd7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 13:47:52 -0700 Subject: [PATCH 02/11] backend - update slack alert_to_webhook_url_map --- litellm/proxy/proxy_server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4f5b647725..89faa491bf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2632,9 +2632,17 @@ class ProxyConfig: if "alert_types" in _general_settings: general_settings["alert_types"] = _general_settings["alert_types"] proxy_logging_obj.alert_types = general_settings["alert_types"] - proxy_logging_obj.slack_alerting_instance.alert_types = general_settings[ - "alert_types" + proxy_logging_obj.slack_alerting_instance.update_values( + alert_types=general_settings["alert_types"] + ) + + if "alert_to_webhook_url" in _general_settings: + general_settings["alert_to_webhook_url"] = _general_settings[ + "alert_to_webhook_url" ] + proxy_logging_obj.slack_alerting_instance.update_values( + alert_to_webhook_url=general_settings["alert_to_webhook_url"] + ) # router settings if llm_router is not None and prisma_client is not None: From 77651a291c52da80b715d2b3d304e77b7728e3da Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 14:00:28 -0700 Subject: [PATCH 03/11] ui - show alert types --- .../src/components/settings.tsx | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 8285f35b80..4f1ff83ca8 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -49,6 +49,12 @@ const Settings: React.FC = ({ const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); const [selectedAlertValues, setSelectedAlertValues] = useState([]); + const alerts_to_UI_NAME: Record = { + "llm_exceptions": "LLM Exceptions", + "llm_too_slow": "LLM Responses Too Slow", + "llm_requests_hanging": "LLM Requests Hanging", + "budget_alerts": "Budget Alerts (API Keys, Users)" + } useEffect(() => { if (!accessToken || !userRole || !userID) { @@ -249,6 +255,25 @@ const Settings: React.FC = ({ Add Callback + + + Alerting + + + + + {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => ( + + + {value} + + + ))} + +
+ + + From 6d43d65e22109cf3098998a91a3381a5d72bc30c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 15:43:54 -0700 Subject: [PATCH 04/11] fix add alert_to_webhook_url to general settings --- litellm/proxy/_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f1d824b864..293d06023a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -720,6 +720,10 @@ class ConfigGeneralSettings(LiteLLMBase): None, description="List of alerting types. By default it is all alerts", ) + alert_to_webhook_url: Optional[Dict] = Field( + None, + description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`", + ) alerting_threshold: Optional[int] = Field( None, From 0c89bdf4835fec4d4660af00c954d516adc076d4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 15:45:08 -0700 Subject: [PATCH 05/11] ui - read/write webhook urls --- .../src/components/settings.tsx | 83 +++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 4f1ff83ca8..7eae54f2e2 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -15,6 +15,7 @@ import { Grid, Button, TextInput, + Switch, Col, } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; @@ -45,10 +46,21 @@ const Settings: React.FC = ({ userID, }) => { const [callbacks, setCallbacks] = useState([]); + const [alerts, setAlerts] = useState([]); const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); const [selectedAlertValues, setSelectedAlertValues] = useState([]); + const [catchAllWebhookURL, setCatchAllWebhookURL] = useState(""); + const [alertToWebhooks, setAlertToWebhooks] = useState([]); + + const [isAlertOn, setIsAlertOn] = useState>({}); // alert_name: boolean + + + + const handleSwitchChange = (value: boolean) => { + //setIsAlertOn(value); + }; const alerts_to_UI_NAME: Record = { "llm_exceptions": "LLM Exceptions", "llm_too_slow": "LLM Responses Too Slow", @@ -62,8 +74,23 @@ const Settings: React.FC = ({ } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); - let callbacks_data = data.data; + let callbacks_data = data.callbacks; setCallbacks(callbacks_data); + + let alerts_data = data.alerts; + console.log("alerts_data", alerts_data); + if (alerts_data) { + if (alerts_data.length > 0) { + let _alert_info = alerts_data[0]; + console.log("_alert_info", _alert_info); + let catch_all_webhook = _alert_info.variables.SLACK_WEBHOOK_URL; + console.log("catch_all_webhook", catch_all_webhook); + setCatchAllWebhookURL(catch_all_webhook); + setAlertToWebhooks(_alert_info.alerts_to_webhook); + } + } + + setAlerts(alerts_data); }); }, [accessToken, userRole, userID]); @@ -84,6 +111,37 @@ const Settings: React.FC = ({ console.log('Selected values:', values); }; + const handleSaveAlerts = () => { + if (!accessToken) { + return; + } + + const updatedAlertToWebhooks = {}; + Object.entries(alerts_to_UI_NAME).forEach(([key, value]) => { + const webhookInput = document.querySelector(`input[name="${key}"]`) as HTMLInputElement; + console.log("key", key); + console.log("webhookInput", webhookInput); + const newWebhookValue = webhookInput?.value || ''; + console.log("newWebhookValue", newWebhookValue); + updatedAlertToWebhooks[key] = newWebhookValue; + }); + + console.log("updatedAlertToWebhooks", updatedAlertToWebhooks); + + const payload = { + general_settings: { + alert_to_webhook_url: updatedAlertToWebhooks, + }, + }; + + try { + setCallbacksCall(accessToken, payload); + } catch (error) { + message.error('Failed to update alerts: ' + error, 20); + } + + message.success('Alerts updated successfully'); + }; const handleSaveChanges = (callback: any) => { if (!accessToken) { return; @@ -98,9 +156,6 @@ const Settings: React.FC = ({ const payload = { environment_variables: updatedVariables, - general_settings: { - alert_types: selectedAlertValues - } }; try { @@ -221,7 +276,7 @@ const Settings: React.FC = ({ ))} - {callback.all_alert_types && ( + {/* {callback.all_alert_types && (
Alerting Types
- )} + )} */} @@ -264,13 +319,29 @@ const Settings: React.FC = ({ {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => ( + + + {value} + + + + + ))} + From 6b125a99defa7be7de15b28b65b370da4d92e42f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 16:00:42 -0700 Subject: [PATCH 06/11] ui - switch alerts on / off --- .../src/components/settings.tsx | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 7eae54f2e2..29069b0547 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -53,13 +53,15 @@ const Settings: React.FC = ({ const [selectedAlertValues, setSelectedAlertValues] = useState([]); const [catchAllWebhookURL, setCatchAllWebhookURL] = useState(""); const [alertToWebhooks, setAlertToWebhooks] = useState([]); + const [activeAlerts, setActiveAlerts] = useState([]); - const [isAlertOn, setIsAlertOn] = useState>({}); // alert_name: boolean - - - const handleSwitchChange = (value: boolean) => { - //setIsAlertOn(value); + const handleSwitchChange = (alertName: string) => { + if (activeAlerts.includes(alertName)) { + setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName)); + } else { + setActiveAlerts([...activeAlerts, alertName]); + } }; const alerts_to_UI_NAME: Record = { "llm_exceptions": "LLM Exceptions", @@ -85,8 +87,12 @@ const Settings: React.FC = ({ console.log("_alert_info", _alert_info); let catch_all_webhook = _alert_info.variables.SLACK_WEBHOOK_URL; console.log("catch_all_webhook", catch_all_webhook); + + let active_alerts = _alert_info.active_alerts; + setActiveAlerts(active_alerts); setCatchAllWebhookURL(catch_all_webhook); setAlertToWebhooks(_alert_info.alerts_to_webhook); + } } @@ -94,6 +100,11 @@ const Settings: React.FC = ({ }); }, [accessToken, userRole, userID]); + + const isAlertOn = (alertName: string) => { + return activeAlerts && activeAlerts.includes(alertName); + } + const handleAddCallback = () => { console.log("Add callback clicked"); setIsModalVisible(true); @@ -127,12 +138,15 @@ const Settings: React.FC = ({ }); console.log("updatedAlertToWebhooks", updatedAlertToWebhooks); - + const payload = { general_settings: { alert_to_webhook_url: updatedAlertToWebhooks, + alert_types: activeAlerts }, }; + + console.log("payload", payload); try { setCallbacksCall(accessToken, payload); @@ -323,8 +337,8 @@ const Settings: React.FC = ({ handleSwitchChange(key)} /> From 6b522885319252cefd356d099084e1fecebf153a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 16:09:28 -0700 Subject: [PATCH 07/11] ui - use tabs for callbacks and alerting --- .../src/components/settings.tsx | 146 ++++++++---------- 1 file changed, 68 insertions(+), 78 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 29069b0547..b74582f67a 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -17,6 +17,11 @@ import { TextInput, Switch, Col, + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; @@ -261,73 +266,62 @@ const Settings: React.FC = ({ return (
- Logging Callbacks - - - - - Callback - Callback Env Vars - - - - {callbacks.map((callback, index) => ( - - - {callback.name} - - -
    - {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( -
  • - {key} - {key === "LANGFUSE_HOST" ? ( -

    default value=https://cloud.langfuse.com

    - ) : ( -
    - )} - -
    • -))} -
- {/* {callback.all_alert_types && ( -
- Alerting Types - -
- )} */} - - - - + + + Logging Callbacks + Alerting + + + + +
+ + + Callback + Callback Env Vars + + + + {callbacks.map((callback, index) => ( + + + {callback.name} + + +
    + {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( +
  • + {key} + {key === "LANGFUSE_HOST" ? ( +

    default value=https://cloud.langfuse.com

    + ) : ( +
    + )} + +
    • ))} - -
- - - + + + + + + ))} + + + + + + - Alerting - + + + @@ -360,6 +354,15 @@ const Settings: React.FC = ({ + + + + + + + + + @@ -379,7 +382,6 @@ const Settings: React.FC = ({ > @@ -407,18 +409,6 @@ const Settings: React.FC = ({ )} - {selectedCallback === 'slack' && ( - - - - )} -
Save
From 557d45b4ec654b67221d7c39966a6dd9c09a4985 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 16:15:39 -0700 Subject: [PATCH 08/11] ui - polish alerts screen --- ui/litellm-dashboard/src/components/settings.tsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index b74582f67a..125bd18a78 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -322,7 +322,15 @@ const Settings: React.FC = ({ + Alerts are only supported for Slack Webhook URLs. Get your webhook urls from here
+ + + + + Slack Webhook URL + + {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => ( @@ -359,11 +367,7 @@ const Settings: React.FC = ({ - - - - - + Date: Thu, 25 Apr 2024 16:29:52 -0700 Subject: [PATCH 09/11] test alerts setup --- ui/litellm-dashboard/src/components/settings.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 125bd18a78..1d015386a1 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -359,6 +359,12 @@ const Settings: React.FC = ({ Save Changes + + + + From ebc27a296998da43ca31bc34def4f181f62fd8ac Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 16:30:05 -0700 Subject: [PATCH 10/11] fix alert success message --- ui/litellm-dashboard/src/components/networking.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 264964a940..db03a2bd00 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1274,7 +1274,7 @@ export const serviceHealthCheck= async (accessToken: String, service: String) => } const data = await response.json(); - message.success(`Test request to ${service} made - check logs on ${service} dashboard!`); + message.success(`Test request to ${service} made - check logs/alerts on ${service} to verify`); // You can add additional logic here based on the response if needed return data; } catch (error) { From 96921864dc7f7e52a509fc2168bfd724323617fe Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 16:33:55 -0700 Subject: [PATCH 11/11] fixes for testing alerting --- litellm/proxy/proxy_server.py | 69 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 89faa491bf..87bf8df5c1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8600,6 +8600,7 @@ async def get_config(): # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) + alerting_data = [] if "slack" in _alerting: _slack_vars = [ "SLACK_WEBHOOK_URL", @@ -8608,7 +8609,8 @@ async def get_config(): for _var in _slack_vars: env_variable = environment_variables.get(_var, None) if env_variable is None: - _slack_env_vars[_var] = None + _value = os.getenv("SLACK_WEBHOOK_URL", None) + _slack_env_vars[_var] = _value else: # decode + decrypt the value decoded_b64 = base64.b64decode(env_variable) @@ -8621,19 +8623,23 @@ async def get_config(): _all_alert_types = ( proxy_logging_obj.slack_alerting_instance._all_possible_alert_types() ) - _data_to_return.append( + _alerts_to_webhook = ( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + ) + alerting_data.append( { "name": "slack", "variables": _slack_env_vars, - "alerting_types": _alerting_types, - "all_alert_types": _all_alert_types, + "active_alerts": _alerting_types, + "alerts_to_webhook": _alerts_to_webhook, } ) _router_settings = llm_router.get_settings() return { "status": "success", - "data": _data_to_return, + "callbacks": _data_to_return, + "alerts": alerting_data, "router_settings": _router_settings, } except Exception as e: @@ -8750,10 +8756,51 @@ async def health_services_endpoint( } if "slack" in general_settings.get("alerting", []): - test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" - await proxy_logging_obj.alerting_handler( - message=test_message, level="Low", alert_type="budget_alerts" - ) + # test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" + # check if user has opted into unique_alert_webhooks + if ( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + is not None + ): + for ( + alert_type + ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url: + """ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + """ + # only test alert if it's in active alert types + if ( + proxy_logging_obj.slack_alerting_instance.alert_types + is not None + and alert_type + not in proxy_logging_obj.slack_alerting_instance.alert_types + ): + continue + test_message = "default test message" + if alert_type == "llm_exceptions": + test_message = f"LLM Exception test alert" + elif alert_type == "llm_too_slow": + test_message = f"LLM Too Slow test alert" + elif alert_type == "llm_requests_hanging": + test_message = f"LLM Requests Hanging test alert" + elif alert_type == "budget_alerts": + test_message = f"Budget Alert test alert" + elif alert_type == "db_exceptions": + test_message = f"DB Exception test alert" + + await proxy_logging_obj.alerting_handler( + message=test_message, level="Low", alert_type=alert_type + ) + else: + await proxy_logging_obj.alerting_handler( + message="This is a test slack alert message", + level="Low", + alert_type="budget_alerts", + ) return { "status": "success", "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel", @@ -8771,7 +8818,7 @@ async def health_services_endpoint( message=getattr(e, "detail", f"Authentication Error({str(e)})"), type="auth_error", param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): raise e @@ -8779,7 +8826,7 @@ async def health_services_endpoint( message="Authentication Error, " + str(e), type="auth_error", param=getattr(e, "param", "None"), - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, )