From 52d7fc22bb1c467ce6fc2b09b60e0b439b90c3d4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 17 Apr 2024 18:16:19 -0700 Subject: [PATCH 1/7] v0 add types of alerts to slack alerting --- litellm/proxy/utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 598a0732b9..cd2d160ea9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -64,6 +64,21 @@ class ProxyLogging: self.cache_control_check = _PROXY_CacheControlCheck() self.alerting: Optional[List] = None self.alerting_threshold: float = 300 # default to 5 min. threshold + self.alert_types: List[ + Literal[ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + ] + ] = [ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + ] def update_values( self, @@ -210,6 +225,8 @@ class ProxyLogging: ): if self.alerting is None: return + if "llm_too_slow" not in self.alert_types: + return time_difference_float, model, api_base, messages = ( self._response_taking_too_long_callback( kwargs=kwargs, @@ -256,6 +273,8 @@ class ProxyLogging: if type == "hanging_request": # Simulate a long-running operation that could take more than 5 minutes + if "llm_requests_hanging" not in self.alert_types: + return await asyncio.sleep( self.alerting_threshold ) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests @@ -304,6 +323,8 @@ class ProxyLogging: if self.alerting is None: # do nothing if alerting is not switched on return + if "budget_alerts" not in self.alert_types: + return _id: str = "default_id" # used for caching if type == "user_and_proxy_budget": user_info = dict(user_info) @@ -460,6 +481,8 @@ class ProxyLogging: Currently only logs exceptions to sentry """ ### ALERTING ### + if "db_exceptions" not in self.alert_types: + return if isinstance(original_exception, HTTPException): if isinstance(original_exception.detail, str): error_message = original_exception.detail @@ -494,6 +517,8 @@ class ProxyLogging: """ ### ALERTING ### + if "llm_exceptions" not in self.alert_types: + return asyncio.create_task( self.alerting_handler( message=f"LLM API call failed: {str(original_exception)}", level="High" From beeee0119969961078aa60ac4849cd87c9886738 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 17 Apr 2024 21:02:10 -0700 Subject: [PATCH 2/7] feat return alert types on /config/get/callback --- litellm/proxy/proxy_server.py | 11 ++++++++++- litellm/proxy/utils.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0c4de59b2e..b2349a6b2b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2283,6 +2283,7 @@ class ProxyConfig: proxy_logging_obj.update_values( alerting=general_settings.get("alerting", None), alerting_threshold=general_settings.get("alerting_threshold", 600), + alert_types=general_settings.get("alert_types", None), redis_cache=redis_usage_cache, ) ### CONNECT TO DATABASE ### @@ -8354,7 +8355,15 @@ async def get_config(): ) _slack_env_vars[_var] = _decrypted_value - _data_to_return.append({"name": "slack", "variables": _slack_env_vars}) + _alerting_types = proxy_logging_obj.alert_types + + _data_to_return.append( + { + "name": "slack", + "variables": _slack_env_vars, + "alerting_types": _alerting_types, + } + ) _router_settings = llm_router.get_settings() return { diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cd2d160ea9..78b12958bb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -85,6 +85,17 @@ class ProxyLogging: alerting: Optional[List], alerting_threshold: Optional[float], redis_cache: Optional[RedisCache], + alert_types: Optional[ + List[ + Literal[ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + ] + ] + ], ): self.alerting = alerting if alerting_threshold is not None: @@ -93,6 +104,9 @@ class ProxyLogging: if redis_cache is not None: self.internal_usage_cache.redis_cache = redis_cache + if alert_types is not None: + self.alert_types = alert_types + def _init_litellm_callbacks(self): print_verbose(f"INITIALIZING LITELLM CALLBACKS!") litellm.callbacks.append(self.max_parallel_request_limiter) From 29b1002b3bf98d3a7bb940a2c00a8078236c925e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 17 Apr 2024 21:26:37 -0700 Subject: [PATCH 3/7] ui - view and set alerting types --- .../src/components/settings.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index c7c60bb448..8ef7b052cb 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -169,6 +169,24 @@ const Settings: React.FC = ({ ))} + {callback.alerting_types && ( +
+ Alerting Types + +
+ )} From a8e2ef79efc58aaddbe4c1cd8fc824d158892214 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 08:51:07 -0700 Subject: [PATCH 4/7] add alert_types to config.yaml --- litellm/proxy/_types.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d0c8eac4ed..25a2670020 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -697,6 +697,21 @@ class ConfigGeneralSettings(LiteLLMBase): None, description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", ) + alert_types: Optional[ + List[ + Literal[ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + ] + ] + ] = Field( + None, + description="List of alerting types. By default it is all alerts", + ) + alerting_threshold: Optional[int] = Field( None, description="sends alerts if requests hang for 5min+", From b7393eb549af9ac3cecf5d53d2d2a2c8217a2e09 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 08:52:55 -0700 Subject: [PATCH 5/7] ui - set selected alerts --- ui/litellm-dashboard/src/components/settings.tsx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 8ef7b052cb..1f9fe39e0f 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -36,6 +36,7 @@ const Settings: React.FC = ({ const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); + const [selectedAlertValues, setSelectedAlertValues] = useState([]); useEffect(() => { if (!accessToken || !userRole || !userID) { @@ -59,6 +60,12 @@ const Settings: React.FC = ({ setSelectedCallback(null); }; + const handleChange = (values) => { + setSelectedAlertValues(values); + // Here, you can perform any additional logic with the selected values + console.log('Selected values:', values); + }; + const handleSaveChanges = (callback: any) => { if (!accessToken) { return; @@ -68,8 +75,14 @@ const Settings: React.FC = ({ Object.entries(callback.variables).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]) ); + console.log("updatedVariables", updatedVariables); + console.log("updateAlertTypes", selectedAlertValues); + const payload = { environment_variables: updatedVariables, + general_settings: { + alert_types: selectedAlertValues + } }; try { @@ -177,6 +190,7 @@ const Settings: React.FC = ({ style={{ width: '100%' }} placeholder="Select Alerting Types" optionLabelProp="label" + onChange={handleChange} defaultValue={callback.alerting_types} > {callback.alerting_types.map((type: string) => ( From 5e0dc573294aa5fd39d73634541365c3bc2e175d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 11:13:05 -0700 Subject: [PATCH 6/7] ui - set `alert_types` --- litellm/proxy/proxy_server.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b2349a6b2b..4cc708602d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2620,6 +2620,9 @@ class ProxyConfig: if "alerting" in _general_settings: general_settings["alerting"] = _general_settings["alerting"] proxy_logging_obj.alerting = general_settings["alerting"] + if "alert_types" in _general_settings: + general_settings["alert_types"] = _general_settings["alert_types"] + proxy_logging_obj.alert_types = general_settings["alert_types"] # router settings _router_settings = config_data.get("router_settings", {}) @@ -8179,10 +8182,12 @@ async def update_config(config_info: ConfigYAML): updated_general_settings = config_info.general_settings.dict( exclude_none=True ) - config["general_settings"] = { - **updated_general_settings, - **config["general_settings"], - } + + _existing_settings = config["general_settings"] + for k, v in updated_general_settings.items(): + # overwrite existing settings with updated values + _existing_settings[k] = v + config["general_settings"] = _existing_settings if config_info.environment_variables is not None: config.setdefault("environment_variables", {}) From 1cda0db2caaa56f881033cacaa176ddb2cc9a792 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 11:40:40 -0700 Subject: [PATCH 7/7] fix - test alerting --- litellm/proxy/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 78b12958bb..179e4273de 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -95,7 +95,7 @@ class ProxyLogging: "db_exceptions", ] ] - ], + ] = None, ): self.alerting = alerting if alerting_threshold is not None: