From f54510b6ee13cb89d3629fa7178a9731de010e97 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Apr 2024 16:42:42 -0700 Subject: [PATCH 1/3] fix(proxy_server.py): fix `/config/update`/ allows updating router config via UI and having the change be propogated across all proxy instances by persisting config changes to the db --- litellm/proxy/_types.py | 3 +- litellm/proxy/proxy_server.py | 116 +++++------------- litellm/router.py | 2 + litellm/types/router.py | 17 +++ ...est_adding_callbacks.py => test_config.py} | 0 .../src/components/general_settings.tsx | 2 +- 6 files changed, 50 insertions(+), 90 deletions(-) rename tests/{test_adding_callbacks.py => test_config.py} (100%) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ca9926cef0..f1d824b864 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4,6 +4,7 @@ import enum from typing import Optional, List, Union, Dict, Literal, Any from datetime import datetime import uuid, json, sys, os +from litellm.types.router import UpdateRouterConfig def hash_token(token: str): @@ -750,7 +751,7 @@ class ConfigYAML(LiteLLMBase): description="litellm Module settings. See __init__.py for all, example litellm.drop_params=True, litellm.set_verbose=True, litellm.api_base, litellm.cache", ) general_settings: Optional[ConfigGeneralSettings] = None - router_settings: Optional[dict] = Field( + router_settings: Optional[UpdateRouterConfig] = Field( None, description="litellm router object settings. See router.py __init__ for all, example router.num_retries=5, router.timeout=5, router.max_retries=5, router.retry_after=5", ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1419a963b9..d840b29c07 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2636,9 +2636,16 @@ class ProxyConfig: ] # router settings - if llm_router is not None: - _router_settings = config_data.get("router_settings", {}) - llm_router.update_settings(**_router_settings) + if llm_router is not None and prisma_client is not None: + db_router_settings = await prisma_client.db.litellm_config.find_first( + where={"param_name": "router_settings"} + ) + if ( + db_router_settings is not None + and db_router_settings.param_value is not None + ): + _router_settings = db_router_settings.param_value + llm_router.update_settings(**_router_settings) async def add_deployment( self, @@ -8406,96 +8413,29 @@ async def update_config(config_info: ConfigYAML): """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client try: - import base64 - # Load existing config - config = await proxy_config.get_config() - verbose_proxy_logger.debug("Loaded config: %s", config) + """ + - Update the ConfigTable DB + - Run 'add_deployment' + """ + if prisma_client is None: + raise Exception("No DB Connected") - # update the general settings - if config_info.general_settings is not None: - config.setdefault("general_settings", {}) - updated_general_settings = config_info.general_settings.dict( - exclude_none=True + updated_settings = config_info.json(exclude_none=True) + updated_settings = prisma_client.jsonify_object(updated_settings) + for k, v in updated_settings.items(): + await prisma_client.db.litellm_config.upsert( + where={"param_name": k}, + data={ + "create": {"param_name": k, "param_value": v}, + "update": {"param_value": v}, + }, ) - _existing_settings = config["general_settings"] - for k, v in updated_general_settings.items(): - # overwrite existing settings with updated values - _existing_settings[k] = v - config["general_settings"] = _existing_settings + await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) - if config_info.environment_variables is not None: - config.setdefault("environment_variables", {}) - _updated_environment_variables = config_info.environment_variables - - # encrypt updated_environment_variables # - for k, v in _updated_environment_variables.items(): - if isinstance(v, str): - encrypted_value = encrypt_value(value=v, master_key=master_key) # type: ignore - _updated_environment_variables[k] = base64.b64encode( - encrypted_value - ).decode("utf-8") - - _existing_env_variables = config["environment_variables"] - - for k, v in _updated_environment_variables.items(): - # overwrite existing env variables with updated values - _existing_env_variables[k] = _updated_environment_variables[k] - - # update the litellm settings - if config_info.litellm_settings is not None: - config.setdefault("litellm_settings", {}) - updated_litellm_settings = config_info.litellm_settings - config["litellm_settings"] = { - **updated_litellm_settings, - **config["litellm_settings"], - } - - # if litellm.success_callback in updated_litellm_settings and config["litellm_settings"] - if ( - "success_callback" in updated_litellm_settings - and "success_callback" in config["litellm_settings"] - ): - - # check both success callback are lists - if isinstance( - config["litellm_settings"]["success_callback"], list - ) and isinstance(updated_litellm_settings["success_callback"], list): - combined_success_callback = ( - config["litellm_settings"]["success_callback"] - + updated_litellm_settings["success_callback"] - ) - combined_success_callback = list(set(combined_success_callback)) - config["litellm_settings"][ - "success_callback" - ] = combined_success_callback - - # router settings - if config_info.router_settings is not None: - config.setdefault("router_settings", {}) - _updated_router_settings = config_info.router_settings - - config["router_settings"] = { - **config["router_settings"], - **_updated_router_settings, - } - - # Save the updated config - await proxy_config.save_config(new_config=config) - - # make sure the change is instantly rolled out for langfuse - if prisma_client is not None: - await proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj - ) - - # Test new connections - ## Slack - if "slack" in config.get("general_settings", {}).get("alerting", []): - await proxy_logging_obj.alerting_handler( - message="This is a test", level="Low" - ) return {"message": "Config updated successfully"} except Exception as e: traceback.print_exc() diff --git a/litellm/router.py b/litellm/router.py index da871dabad..2a710f5cfc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2553,6 +2553,8 @@ class Router: "timeout", "max_retries", "retry_after", + "fallbacks", + "context_window_fallbacks", ] _int_settings = [ diff --git a/litellm/types/router.py b/litellm/types/router.py index 961f20a91d..c5ec47091a 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -48,6 +48,23 @@ class RouterConfig(BaseModel): protected_namespaces = () +class UpdateRouterConfig(BaseModel): + """ + Set of params that you can modify via `router.update_settings()`. + """ + + routing_strategy_args: Optional[dict] = None + routing_strategy: Optional[str] = None + allowed_fails: Optional[int] = None + cooldown_time: Optional[float] = None + num_retries: Optional[int] = None + timeout: Optional[float] = None + max_retries: Optional[int] = None + retry_after: Optional[float] = None + fallbacks: Optional[List[dict]] = None + context_window_fallbacks: Optional[List[dict]] = None + + class ModelInfo(BaseModel): id: Optional[ str diff --git a/tests/test_adding_callbacks.py b/tests/test_config.py similarity index 100% rename from tests/test_adding_callbacks.py rename to tests/test_config.py diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index e227568d09..ace755a2cd 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -125,7 +125,7 @@ const GeneralSettings: React.FC = ({ - {Object.entries(routerSettings).map(([param, value]) => ( + {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks").map(([param, value]) => ( {param} From 94cbe5516e6771672f16a739ad367011261484e6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Apr 2024 17:33:00 -0700 Subject: [PATCH 2/3] feat(router.py): support mock testing fallbacks flag on router --- litellm/router.py | 6 ++ .../src/components/general_settings.tsx | 92 ++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 2a710f5cfc..e6db92397b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1309,12 +1309,18 @@ class Router: Try calling the function_with_retries If it fails after num_retries, fall back to another model group """ + mock_testing_fallbacks = kwargs.get("mock_testing_fallbacks", None) model_group = kwargs.get("model") fallbacks = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.get( "context_window_fallbacks", self.context_window_fallbacks ) try: + if mock_testing_fallbacks is not None and mock_testing_fallbacks == True: + raise Exception( + f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}" + ) + response = await self.async_function_with_retries(*args, **kwargs) verbose_router_logger.debug(f"Async Response: {response}") return response diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index ace755a2cd..cc88bb1ee6 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -20,8 +20,10 @@ import { import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; +import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon } from "@heroicons/react/outline"; import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; import AddFallbacks from "./add_fallbacks" +import openai from "openai"; interface GeneralSettingsPageProps { accessToken: string | null; @@ -30,6 +32,44 @@ interface GeneralSettingsPageProps { modelData: any } +async function testFallbackModelResponse( + selectedModel: string, + accessToken: string +) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ + apiKey: accessToken, // Replace with your OpenAI API key + baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL + dangerouslyAllowBrowser: true, // using a temporary litellm proxy key + }); + + try { + const response = await client.chat.completions.create({ + model: selectedModel, + messages: [ + { + role: "user", + content: "Hi, this is a test message", + }, + ], + }); + + message.success( + + Test model={selectedModel}, received model={responseModel}. + See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl + + ); + } catch (error) { + message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); + } +} + const GeneralSettings: React.FC = ({ accessToken, userRole, @@ -73,6 +113,38 @@ const GeneralSettings: React.FC = ({ setSelectedCallback(null); }; + const deleteFallbacks = async (key: string) => { + /** + * pop the key from the Object, if it exists + */ + if (!accessToken) { + return; + } + + console.log(`received key: ${key}`) + console.log(`routerSettings['fallbacks']: ${routerSettings['fallbacks']}`) + + routerSettings["fallbacks"].map((dict: { [key: string]: any }) => { + // Check if the dictionary has the specified key and delete it if present + if (key in dict) { + delete dict[key]; + } + return dict; // Return the updated dictionary + }); + + const payload = { + router_settings: routerSettings + }; + + try { + await setCallbacksCall(accessToken, payload); + setRouterSettings({ ...routerSettings }); + message.success("Router settings updated successfully"); + } catch (error) { + message.error("Failed to update router settings: " + error, 20); + } + } + const handleSaveChanges = (router_settings: any) => { if (!accessToken) { return; @@ -81,7 +153,13 @@ const GeneralSettings: React.FC = ({ console.log("router_settings", router_settings); const updatedVariables = Object.fromEntries( - Object.entries(router_settings).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]) + Object.entries(router_settings).map(([key, value]) => { + if (key === 'routing_strategy_args' && typeof value === 'string') { + return [key, JSON.parse(value as string)]; + } else { + return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]; + } + }) ); console.log("updatedVariables", updatedVariables); @@ -168,6 +246,18 @@ const GeneralSettings: React.FC = ({ {key} {Array.isArray(value) ? value.join(', ') : value} + + + + + deleteFallbacks(key)} + /> + )) ) From 5650e8ea444038d0968fc1cbf6245fc4f02ff0d0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Apr 2024 20:13:10 -0700 Subject: [PATCH 3/3] feat(router.py): support mock testing fallbacks flag allow user to test if fallbacks work as expected with a `mock_testing_fallbacks = True` flag set during a call --- docs/my-website/docs/proxy/reliability.md | 15 +++++++++++++++ litellm/proxy/proxy_server.py | 7 ++++--- litellm/router.py | 7 +++++-- .../src/components/general_settings.tsx | 11 +++++------ .../src/components/model_dashboard.tsx | 3 +++ 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index 51e90fe39f..bd04216dd1 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -136,6 +136,21 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` +### Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "model": "zephyr-beta", # 👈 MODEL NAME to fallback from + "messages": [ + {"role": "user", "content": "what color is red"} + ], + "mock_testing_fallbacks": true + }' +``` + ## Advanced - Context Window Fallbacks **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d840b29c07..9bcc280b31 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2521,9 +2521,10 @@ class ProxyConfig: # decode base64 decoded_b64 = base64.b64decode(v) # decrypt value - _litellm_params[k] = decrypt_value( - value=decoded_b64, master_key=master_key - ) + _value = decrypt_value(value=decoded_b64, master_key=master_key) + # sanity check if string > size 0 + if len(_value) > 0: + _litellm_params[k] = _value _litellm_params = LiteLLM_Params(**_litellm_params) else: verbose_proxy_logger.error( diff --git a/litellm/router.py b/litellm/router.py index e6db92397b..672eb097de 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1309,7 +1309,7 @@ class Router: Try calling the function_with_retries If it fails after num_retries, fall back to another model group """ - mock_testing_fallbacks = kwargs.get("mock_testing_fallbacks", None) + mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None) model_group = kwargs.get("model") fallbacks = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.get( @@ -1369,7 +1369,10 @@ class Router: elif fallbacks is not None: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") for item in fallbacks: - if list(item.keys())[0] == model_group: + key_list = list(item.keys()) + if len(key_list) == 0: + continue + if key_list[0] == model_group: fallback_model_group = item[model_group] break if fallback_model_group is None: diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index cc88bb1ee6..f437306108 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -57,11 +57,12 @@ async function testFallbackModelResponse( content: "Hi, this is a test message", }, ], + mock_testing_fallbacks: true }); message.success( - Test model={selectedModel}, received model={responseModel}. + Test model={selectedModel}, received model={response.model}. See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl ); @@ -154,14 +155,12 @@ const GeneralSettings: React.FC = ({ const updatedVariables = Object.fromEntries( Object.entries(router_settings).map(([key, value]) => { - if (key === 'routing_strategy_args' && typeof value === 'string') { - return [key, JSON.parse(value as string)]; - } else { + if (key !== 'routing_strategy_args') { return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]; } - }) + return null; + }).filter(entry => entry !== null) as Iterable<[string, unknown]> ); - console.log("updatedVariables", updatedVariables); const payload = { diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 89b573f771..c3ea6f5aa9 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -106,6 +106,9 @@ const handleSubmit = async (formValues: Record, accessToken: string litellmParamsObj["model"] = litellm_model let modelName: string = ""; for (const [key, value] of Object.entries(formValues)) { + if (value === '') { + continue; + } if (key == "model_name") { modelName = modelName + value }