diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md
index 51e90fe39f..bd04216dd1 100644
--- a/docs/my-website/docs/proxy/reliability.md
+++ b/docs/my-website/docs/proxy/reliability.md
@@ -136,6 +136,21 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
'
```
+### Test it!
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data-raw '{
+ "model": "zephyr-beta", # 👈 MODEL NAME to fallback from
+ "messages": [
+ {"role": "user", "content": "what color is red"}
+ ],
+ "mock_testing_fallbacks": true
+ }'
+```
+
## Advanced - Context Window Fallbacks
**Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**.
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index ca9926cef0..f1d824b864 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -4,6 +4,7 @@ import enum
from typing import Optional, List, Union, Dict, Literal, Any
from datetime import datetime
import uuid, json, sys, os
+from litellm.types.router import UpdateRouterConfig
def hash_token(token: str):
@@ -750,7 +751,7 @@ class ConfigYAML(LiteLLMBase):
description="litellm Module settings. See __init__.py for all, example litellm.drop_params=True, litellm.set_verbose=True, litellm.api_base, litellm.cache",
)
general_settings: Optional[ConfigGeneralSettings] = None
- router_settings: Optional[dict] = Field(
+ router_settings: Optional[UpdateRouterConfig] = Field(
None,
description="litellm router object settings. See router.py __init__ for all, example router.num_retries=5, router.timeout=5, router.max_retries=5, router.retry_after=5",
)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 1419a963b9..9bcc280b31 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -2521,9 +2521,10 @@ class ProxyConfig:
# decode base64
decoded_b64 = base64.b64decode(v)
# decrypt value
- _litellm_params[k] = decrypt_value(
- value=decoded_b64, master_key=master_key
- )
+ _value = decrypt_value(value=decoded_b64, master_key=master_key)
+ # sanity check if string > size 0
+ if len(_value) > 0:
+ _litellm_params[k] = _value
_litellm_params = LiteLLM_Params(**_litellm_params)
else:
verbose_proxy_logger.error(
@@ -2636,9 +2637,16 @@ class ProxyConfig:
]
# router settings
- if llm_router is not None:
- _router_settings = config_data.get("router_settings", {})
- llm_router.update_settings(**_router_settings)
+ if llm_router is not None and prisma_client is not None:
+ db_router_settings = await prisma_client.db.litellm_config.find_first(
+ where={"param_name": "router_settings"}
+ )
+ if (
+ db_router_settings is not None
+ and db_router_settings.param_value is not None
+ ):
+ _router_settings = db_router_settings.param_value
+ llm_router.update_settings(**_router_settings)
async def add_deployment(
self,
@@ -8406,96 +8414,29 @@ async def update_config(config_info: ConfigYAML):
"""
global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client
try:
- import base64
- # Load existing config
- config = await proxy_config.get_config()
- verbose_proxy_logger.debug("Loaded config: %s", config)
+ """
+ - Update the ConfigTable DB
+ - Run 'add_deployment'
+ """
+ if prisma_client is None:
+ raise Exception("No DB Connected")
- # update the general settings
- if config_info.general_settings is not None:
- config.setdefault("general_settings", {})
- updated_general_settings = config_info.general_settings.dict(
- exclude_none=True
+ updated_settings = config_info.json(exclude_none=True)
+ updated_settings = prisma_client.jsonify_object(updated_settings)
+ for k, v in updated_settings.items():
+ await prisma_client.db.litellm_config.upsert(
+ where={"param_name": k},
+ data={
+ "create": {"param_name": k, "param_value": v},
+ "update": {"param_value": v},
+ },
)
- _existing_settings = config["general_settings"]
- for k, v in updated_general_settings.items():
- # overwrite existing settings with updated values
- _existing_settings[k] = v
- config["general_settings"] = _existing_settings
+ await proxy_config.add_deployment(
+ prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
+ )
- if config_info.environment_variables is not None:
- config.setdefault("environment_variables", {})
- _updated_environment_variables = config_info.environment_variables
-
- # encrypt updated_environment_variables #
- for k, v in _updated_environment_variables.items():
- if isinstance(v, str):
- encrypted_value = encrypt_value(value=v, master_key=master_key) # type: ignore
- _updated_environment_variables[k] = base64.b64encode(
- encrypted_value
- ).decode("utf-8")
-
- _existing_env_variables = config["environment_variables"]
-
- for k, v in _updated_environment_variables.items():
- # overwrite existing env variables with updated values
- _existing_env_variables[k] = _updated_environment_variables[k]
-
- # update the litellm settings
- if config_info.litellm_settings is not None:
- config.setdefault("litellm_settings", {})
- updated_litellm_settings = config_info.litellm_settings
- config["litellm_settings"] = {
- **updated_litellm_settings,
- **config["litellm_settings"],
- }
-
- # if litellm.success_callback in updated_litellm_settings and config["litellm_settings"]
- if (
- "success_callback" in updated_litellm_settings
- and "success_callback" in config["litellm_settings"]
- ):
-
- # check both success callback are lists
- if isinstance(
- config["litellm_settings"]["success_callback"], list
- ) and isinstance(updated_litellm_settings["success_callback"], list):
- combined_success_callback = (
- config["litellm_settings"]["success_callback"]
- + updated_litellm_settings["success_callback"]
- )
- combined_success_callback = list(set(combined_success_callback))
- config["litellm_settings"][
- "success_callback"
- ] = combined_success_callback
-
- # router settings
- if config_info.router_settings is not None:
- config.setdefault("router_settings", {})
- _updated_router_settings = config_info.router_settings
-
- config["router_settings"] = {
- **config["router_settings"],
- **_updated_router_settings,
- }
-
- # Save the updated config
- await proxy_config.save_config(new_config=config)
-
- # make sure the change is instantly rolled out for langfuse
- if prisma_client is not None:
- await proxy_config.add_deployment(
- prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
- )
-
- # Test new connections
- ## Slack
- if "slack" in config.get("general_settings", {}).get("alerting", []):
- await proxy_logging_obj.alerting_handler(
- message="This is a test", level="Low"
- )
return {"message": "Config updated successfully"}
except Exception as e:
traceback.print_exc()
diff --git a/litellm/router.py b/litellm/router.py
index da871dabad..672eb097de 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1309,12 +1309,18 @@ class Router:
Try calling the function_with_retries
If it fails after num_retries, fall back to another model group
"""
+ mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None)
model_group = kwargs.get("model")
fallbacks = kwargs.get("fallbacks", self.fallbacks)
context_window_fallbacks = kwargs.get(
"context_window_fallbacks", self.context_window_fallbacks
)
try:
+ if mock_testing_fallbacks is not None and mock_testing_fallbacks == True:
+ raise Exception(
+ f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}"
+ )
+
response = await self.async_function_with_retries(*args, **kwargs)
verbose_router_logger.debug(f"Async Response: {response}")
return response
@@ -1363,7 +1369,10 @@ class Router:
elif fallbacks is not None:
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
for item in fallbacks:
- if list(item.keys())[0] == model_group:
+ key_list = list(item.keys())
+ if len(key_list) == 0:
+ continue
+ if key_list[0] == model_group:
fallback_model_group = item[model_group]
break
if fallback_model_group is None:
@@ -2553,6 +2562,8 @@ class Router:
"timeout",
"max_retries",
"retry_after",
+ "fallbacks",
+ "context_window_fallbacks",
]
_int_settings = [
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 961f20a91d..c5ec47091a 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -48,6 +48,23 @@ class RouterConfig(BaseModel):
protected_namespaces = ()
+class UpdateRouterConfig(BaseModel):
+ """
+ Set of params that you can modify via `router.update_settings()`.
+ """
+
+ routing_strategy_args: Optional[dict] = None
+ routing_strategy: Optional[str] = None
+ allowed_fails: Optional[int] = None
+ cooldown_time: Optional[float] = None
+ num_retries: Optional[int] = None
+ timeout: Optional[float] = None
+ max_retries: Optional[int] = None
+ retry_after: Optional[float] = None
+ fallbacks: Optional[List[dict]] = None
+ context_window_fallbacks: Optional[List[dict]] = None
+
+
class ModelInfo(BaseModel):
id: Optional[
str
diff --git a/tests/test_adding_callbacks.py b/tests/test_config.py
similarity index 100%
rename from tests/test_adding_callbacks.py
rename to tests/test_config.py
diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx
index e227568d09..f437306108 100644
--- a/ui/litellm-dashboard/src/components/general_settings.tsx
+++ b/ui/litellm-dashboard/src/components/general_settings.tsx
@@ -20,8 +20,10 @@ import {
import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
+import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon } from "@heroicons/react/outline";
import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
import AddFallbacks from "./add_fallbacks"
+import openai from "openai";
interface GeneralSettingsPageProps {
accessToken: string | null;
@@ -30,6 +32,45 @@ interface GeneralSettingsPageProps {
modelData: any
}
+async function testFallbackModelResponse(
+ selectedModel: string,
+ accessToken: string
+) {
+ // base url should be the current base_url
+ const isLocal = process.env.NODE_ENV === "development";
+ console.log("isLocal:", isLocal);
+ const proxyBaseUrl = isLocal
+ ? "http://localhost:4000"
+ : window.location.origin;
+ const client = new openai.OpenAI({
+ apiKey: accessToken, // Replace with your OpenAI API key
+ baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
+ dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
+ });
+
+ try {
+ const response = await client.chat.completions.create({
+ model: selectedModel,
+ messages: [
+ {
+ role: "user",
+ content: "Hi, this is a test message",
+ },
+ ],
+ mock_testing_fallbacks: true
+ });
+
+ message.success(
+
+ Test model={selectedModel}, received model={response.model}.
+ See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl
+
+ );
+ } catch (error) {
+ message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
+ }
+}
+
const GeneralSettings: React.FC = ({
accessToken,
userRole,
@@ -73,6 +114,38 @@ const GeneralSettings: React.FC = ({
setSelectedCallback(null);
};
+ const deleteFallbacks = async (key: string) => {
+ /**
+ * pop the key from the Object, if it exists
+ */
+ if (!accessToken) {
+ return;
+ }
+
+ console.log(`received key: ${key}`)
+ console.log(`routerSettings['fallbacks']: ${routerSettings['fallbacks']}`)
+
+ routerSettings["fallbacks"].map((dict: { [key: string]: any }) => {
+ // Check if the dictionary has the specified key and delete it if present
+ if (key in dict) {
+ delete dict[key];
+ }
+ return dict; // Return the updated dictionary
+ });
+
+ const payload = {
+ router_settings: routerSettings
+ };
+
+ try {
+ await setCallbacksCall(accessToken, payload);
+ setRouterSettings({ ...routerSettings });
+ message.success("Router settings updated successfully");
+ } catch (error) {
+ message.error("Failed to update router settings: " + error, 20);
+ }
+ }
+
const handleSaveChanges = (router_settings: any) => {
if (!accessToken) {
return;
@@ -81,9 +154,13 @@ const GeneralSettings: React.FC = ({
console.log("router_settings", router_settings);
const updatedVariables = Object.fromEntries(
- Object.entries(router_settings).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value])
+ Object.entries(router_settings).map(([key, value]) => {
+ if (key !== 'routing_strategy_args') {
+ return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value];
+ }
+ return null;
+ }).filter(entry => entry !== null) as Iterable<[string, unknown]>
);
-
console.log("updatedVariables", updatedVariables);
const payload = {
@@ -125,7 +202,7 @@ const GeneralSettings: React.FC = ({
- {Object.entries(routerSettings).map(([param, value]) => (
+ {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks").map(([param, value]) => (
{param}
@@ -168,6 +245,18 @@ const GeneralSettings: React.FC = ({
{key}
{Array.isArray(value) ? value.join(', ') : value}
+
+
+
+
+ deleteFallbacks(key)}
+ />
+
))
)
diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx
index 89b573f771..c3ea6f5aa9 100644
--- a/ui/litellm-dashboard/src/components/model_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx
@@ -106,6 +106,9 @@ const handleSubmit = async (formValues: Record, accessToken: string
litellmParamsObj["model"] = litellm_model
let modelName: string = "";
for (const [key, value] of Object.entries(formValues)) {
+ if (value === '') {
+ continue;
+ }
if (key == "model_name") {
modelName = modelName + value
}