Merge pull request #3282 from BerriAI/litellm_config_update_fixes

fix(proxy_server.py): fix `/config/update`

docs/my-website/docs/proxy/reliability.md

@@ -136,6 +136,21 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 '
 ```
 
+### Test it!
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+     --header 'Content-Type: application/json' \
+     --data-raw '{
+        "model": "zephyr-beta", # 👈 MODEL NAME to fallback from
+        "messages": [
+            {"role": "user", "content": "what color is red"}
+        ],
+        "mock_testing_fallbacks": true
+     }'
+```
+
 ## Advanced - Context Window Fallbacks 
 
 **Before call is made** check if a call is within model context window with  **`enable_pre_call_checks: true`**.

litellm/proxy/_types.py

@@ -4,6 +4,7 @@ import enum
 from typing import Optional, List, Union, Dict, Literal, Any
 from datetime import datetime
 import uuid, json, sys, os
+from litellm.types.router import UpdateRouterConfig
 
 
 def hash_token(token: str):
@@ -750,7 +751,7 @@ class ConfigYAML(LiteLLMBase):
         description="litellm Module settings. See __init__.py for all, example litellm.drop_params=True, litellm.set_verbose=True, litellm.api_base, litellm.cache",
     )
     general_settings: Optional[ConfigGeneralSettings] = None
-    router_settings: Optional[dict] = Field(
+    router_settings: Optional[UpdateRouterConfig] = Field(
         None,
         description="litellm router object settings. See router.py __init__ for all, example router.num_retries=5, router.timeout=5, router.max_retries=5, router.retry_after=5",
     )

litellm/proxy/proxy_server.py

@@ -2521,9 +2521,10 @@ class ProxyConfig:
                         # decode base64
                         decoded_b64 = base64.b64decode(v)
                         # decrypt value
-                        _litellm_params[k] = decrypt_value(
-                            value=decoded_b64, master_key=master_key
-                        )
+                        _value = decrypt_value(value=decoded_b64, master_key=master_key)
+                        # sanity check if string > size 0
+                        if len(_value) > 0:
+                            _litellm_params[k] = _value
                 _litellm_params = LiteLLM_Params(**_litellm_params)
             else:
                 verbose_proxy_logger.error(
@@ -2636,9 +2637,16 @@ class ProxyConfig:
             ]
 
         # router settings
-        if llm_router is not None:
-            _router_settings = config_data.get("router_settings", {})
-            llm_router.update_settings(**_router_settings)
+        if llm_router is not None and prisma_client is not None:
+            db_router_settings = await prisma_client.db.litellm_config.find_first(
+                where={"param_name": "router_settings"}
+            )
+            if (
+                db_router_settings is not None
+                and db_router_settings.param_value is not None
+            ):
+                _router_settings = db_router_settings.param_value
+                llm_router.update_settings(**_router_settings)
 
     async def add_deployment(
         self,
@@ -8406,96 +8414,29 @@ async def update_config(config_info: ConfigYAML):
     """
     global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client
     try:
-        import base64
 
-        # Load existing config
-        config = await proxy_config.get_config()
-        verbose_proxy_logger.debug("Loaded config: %s", config)
+        """
+        - Update the ConfigTable DB
+        - Run 'add_deployment'
+        """
+        if prisma_client is None:
+            raise Exception("No DB Connected")
 
-        # update the general settings
-        if config_info.general_settings is not None:
-            config.setdefault("general_settings", {})
-            updated_general_settings = config_info.general_settings.dict(
-                exclude_none=True
+        updated_settings = config_info.json(exclude_none=True)
+        updated_settings = prisma_client.jsonify_object(updated_settings)
+        for k, v in updated_settings.items():
+            await prisma_client.db.litellm_config.upsert(
+                where={"param_name": k},
+                data={
+                    "create": {"param_name": k, "param_value": v},
+                    "update": {"param_value": v},
+                },
             )
 
-            _existing_settings = config["general_settings"]
-            for k, v in updated_general_settings.items():
-                # overwrite existing settings with updated values
-                _existing_settings[k] = v
-            config["general_settings"] = _existing_settings
+        await proxy_config.add_deployment(
+            prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
+        )
 
-        if config_info.environment_variables is not None:
-            config.setdefault("environment_variables", {})
-            _updated_environment_variables = config_info.environment_variables
-
-            # encrypt updated_environment_variables #
-            for k, v in _updated_environment_variables.items():
-                if isinstance(v, str):
-                    encrypted_value = encrypt_value(value=v, master_key=master_key)  # type: ignore
-                    _updated_environment_variables[k] = base64.b64encode(
-                        encrypted_value
-                    ).decode("utf-8")
-
-            _existing_env_variables = config["environment_variables"]
-
-            for k, v in _updated_environment_variables.items():
-                # overwrite existing env variables with updated values
-                _existing_env_variables[k] = _updated_environment_variables[k]
-
-        # update the litellm settings
-        if config_info.litellm_settings is not None:
-            config.setdefault("litellm_settings", {})
-            updated_litellm_settings = config_info.litellm_settings
-            config["litellm_settings"] = {
-                **updated_litellm_settings,
-                **config["litellm_settings"],
-            }
-
-            # if litellm.success_callback in updated_litellm_settings and config["litellm_settings"]
-            if (
-                "success_callback" in updated_litellm_settings
-                and "success_callback" in config["litellm_settings"]
-            ):
-
-                # check both success callback are lists
-                if isinstance(
-                    config["litellm_settings"]["success_callback"], list
-                ) and isinstance(updated_litellm_settings["success_callback"], list):
-                    combined_success_callback = (
-                        config["litellm_settings"]["success_callback"]
-                        + updated_litellm_settings["success_callback"]
-                    )
-                    combined_success_callback = list(set(combined_success_callback))
-                    config["litellm_settings"][
-                        "success_callback"
-                    ] = combined_success_callback
-
-        # router settings
-        if config_info.router_settings is not None:
-            config.setdefault("router_settings", {})
-            _updated_router_settings = config_info.router_settings
-
-            config["router_settings"] = {
-                **config["router_settings"],
-                **_updated_router_settings,
-            }
-
-        # Save the updated config
-        await proxy_config.save_config(new_config=config)
-
-        # make sure the change is instantly rolled out for langfuse
-        if prisma_client is not None:
-            await proxy_config.add_deployment(
-                prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
-            )
-
-        # Test new connections
-        ## Slack
-        if "slack" in config.get("general_settings", {}).get("alerting", []):
-            await proxy_logging_obj.alerting_handler(
-                message="This is a test", level="Low"
-            )
         return {"message": "Config updated successfully"}
     except Exception as e:
         traceback.print_exc()

litellm/router.py

@@ -1309,12 +1309,18 @@ class Router:
         Try calling the function_with_retries
         If it fails after num_retries, fall back to another model group
         """
+        mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None)
         model_group = kwargs.get("model")
         fallbacks = kwargs.get("fallbacks", self.fallbacks)
         context_window_fallbacks = kwargs.get(
             "context_window_fallbacks", self.context_window_fallbacks
         )
         try:
+            if mock_testing_fallbacks is not None and mock_testing_fallbacks == True:
+                raise Exception(
+                    f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}"
+                )
+
             response = await self.async_function_with_retries(*args, **kwargs)
             verbose_router_logger.debug(f"Async Response: {response}")
             return response
@@ -1363,7 +1369,10 @@ class Router:
                 elif fallbacks is not None:
                     verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
                     for item in fallbacks:
-                        if list(item.keys())[0] == model_group:
+                        key_list = list(item.keys())
+                        if len(key_list) == 0:
+                            continue
+                        if key_list[0] == model_group:
                             fallback_model_group = item[model_group]
                             break
                     if fallback_model_group is None:
@@ -2553,6 +2562,8 @@ class Router:
             "timeout",
             "max_retries",
             "retry_after",
+            "fallbacks",
+            "context_window_fallbacks",
         ]
 
         _int_settings = [

litellm/types/router.py

@@ -48,6 +48,23 @@ class RouterConfig(BaseModel):
         protected_namespaces = ()
 
 
+class UpdateRouterConfig(BaseModel):
+    """
+    Set of params that you can modify via `router.update_settings()`.
+    """
+
+    routing_strategy_args: Optional[dict] = None
+    routing_strategy: Optional[str] = None
+    allowed_fails: Optional[int] = None
+    cooldown_time: Optional[float] = None
+    num_retries: Optional[int] = None
+    timeout: Optional[float] = None
+    max_retries: Optional[int] = None
+    retry_after: Optional[float] = None
+    fallbacks: Optional[List[dict]] = None
+    context_window_fallbacks: Optional[List[dict]] = None
+
+
 class ModelInfo(BaseModel):
     id: Optional[
         str

ui/litellm-dashboard/src/components/general_settings.tsx

@@ -20,8 +20,10 @@ import {
 import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
 import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
 import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
+import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon } from "@heroicons/react/outline";
 import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
 import AddFallbacks from "./add_fallbacks"
+import openai from "openai";
 
 interface GeneralSettingsPageProps {
   accessToken: string | null;
@@ -30,6 +32,45 @@ interface GeneralSettingsPageProps {
   modelData: any
 }
 
+async function testFallbackModelResponse(
+  selectedModel: string,
+  accessToken: string
+) {
+  // base url should be the current base_url
+  const isLocal = process.env.NODE_ENV === "development";
+  console.log("isLocal:", isLocal);
+  const proxyBaseUrl = isLocal
+    ? "http://localhost:4000"
+    : window.location.origin;
+  const client = new openai.OpenAI({
+    apiKey: accessToken, // Replace with your OpenAI API key
+    baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
+    dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
+  });
+
+  try {
+    const response = await client.chat.completions.create({
+      model: selectedModel,
+      messages: [
+        {
+          role: "user",
+          content: "Hi, this is a test message",
+        },
+      ],
+      mock_testing_fallbacks: true
+    });
+
+    message.success(
+      <span>
+        Test model=<strong>{selectedModel}</strong>, received model=<strong>{response.model}</strong>. 
+        See <a href="#" onClick={() => window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl</a>
+      </span>
+    );
+  } catch (error) {
+    message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
+  }
+}
+
 const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
   accessToken,
   userRole,
@@ -73,6 +114,38 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
     setSelectedCallback(null);
   };
 
+  const deleteFallbacks = async (key: string) => {
+    /**
+     * pop the key from the Object, if it exists
+     */
+    if (!accessToken) {
+      return;
+    }
+
+    console.log(`received key: ${key}`)
+    console.log(`routerSettings['fallbacks']: ${routerSettings['fallbacks']}`)
+
+    routerSettings["fallbacks"].map((dict: { [key: string]: any }) => {
+      // Check if the dictionary has the specified key and delete it if present
+      if (key in dict) {
+        delete dict[key];
+      }
+      return dict; // Return the updated dictionary
+    });
+
+    const payload = {
+      router_settings: routerSettings
+    };
+
+    try {
+      await setCallbacksCall(accessToken, payload);
+      setRouterSettings({ ...routerSettings });
+      message.success("Router settings updated successfully");
+    } catch (error) {
+      message.error("Failed to update router settings: " + error, 20);
+    }
+  }
+
   const handleSaveChanges = (router_settings: any) => {
     if (!accessToken) {
       return;
@@ -81,9 +154,13 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
     console.log("router_settings", router_settings);
 
     const updatedVariables = Object.fromEntries(
-      Object.entries(router_settings).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value])
+      Object.entries(router_settings).map(([key, value]) => {
+        if (key !== 'routing_strategy_args') {
+          return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value];
+        }
+        return null;
+      }).filter(entry => entry !== null) as Iterable<[string, unknown]>
     );
-
     console.log("updatedVariables", updatedVariables);
 
     const payload = {
@@ -125,7 +202,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
               </TableRow>
             </TableHead>
             <TableBody>
-              {Object.entries(routerSettings).map(([param, value]) => (
+              {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks").map(([param, value]) => (
                 <TableRow key={param}>
                   <TableCell>
                     <Text>{param}</Text>
@@ -168,6 +245,18 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
                 <TableRow key={index.toString() + key}>
                   <TableCell>{key}</TableCell>
                   <TableCell>{Array.isArray(value) ? value.join(', ') : value}</TableCell>
+                  <TableCell>
+                    <Button onClick={() => testFallbackModelResponse(key, accessToken)}>
+                      Test Fallback
+                    </Button>
+                  </TableCell>
+                  <TableCell>
+                    <Icon
+                        icon={TrashIcon}
+                        size="sm"
+                        onClick={() => deleteFallbacks(key)}
+                      />
+                  </TableCell>
                 </TableRow>
               ))
             )

ui/litellm-dashboard/src/components/model_dashboard.tsx

@@ -106,6 +106,9 @@ const handleSubmit = async (formValues: Record<string, any>, accessToken: string
       litellmParamsObj["model"] = litellm_model
       let modelName: string  = "";
       for (const [key, value] of Object.entries(formValues)) {
+        if (value === '') {
+          continue;
+        }
         if (key == "model_name") {
           modelName = modelName + value
         }