diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7103ffd457..624dea84e1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7073,7 +7073,8 @@ "output_cost_per_token": 6e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/Qwen/Qwen3-14B": { "max_tokens": 40960, @@ -18620,7 +18621,8 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.3e-07, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3-coder": { "input_cost_per_token": 1e-06, @@ -21143,16 +21145,6 @@ "mode": "chat", "output_cost_per_token": 2.4e-07 }, - "vercel_ai_gateway/glm-4.6": { - "litellm_provider": "vercel_ai_gateway", - "cache_read_input_token_cost": 1.1e-07, - "input_cost_per_token": 6e-07, - "max_input_tokens": 200000, - "max_output_tokens": 200000, - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 2.2e-06 - }, "vercel_ai_gateway/alibaba/qwen-3-235b": { "input_cost_per_token": 2e-07, "litellm_provider": "vercel_ai_gateway", @@ -21986,6 +21978,20 @@ "mode": "chat", "output_cost_per_token": 1.1e-06 }, + "vercel_ai_gateway/zai/glm-4.6": { + "litellm_provider": "vercel_ai_gateway", + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 4.5e-07, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 1.8e-06, + "source": "https://vercel.com/ai-gateway/models/glm-4.6", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true + }, "vertex_ai/claude-3-5-haiku": { "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py new file mode 100644 index 0000000000..167160c72d --- /dev/null +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -0,0 +1,122 @@ +""" +ROUTER SETTINGS MANAGEMENT + +Endpoints for accessing router configuration and metadata + +GET /router/settings - Get router configuration including available routing strategies +""" + +import inspect +from typing import Any, Dict, List, get_args + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.router import Router +from litellm.types.management_endpoints import ( + ROUTER_SETTINGS_FIELDS, + ROUTING_STRATEGY_DESCRIPTIONS, + RouterSettingsField, +) + +router = APIRouter() + + +class RouterSettingsResponse(BaseModel): + fields: List[RouterSettingsField] = Field( + description="List of all configurable router settings with metadata" + ) + current_values: Dict[str, Any] = Field( + description="Current values of router settings" + ) + routing_strategy_descriptions: Dict[str, str] = Field( + description="Descriptions for each routing strategy option" + ) + + +def _get_routing_strategies_from_router_class() -> List[str]: + """ + Dynamically extract routing strategies from the Router class __init__ method. + """ + # Get the __init__ signature + sig = inspect.signature(Router.__init__) + + # Get the routing_strategy parameter + routing_strategy_param = sig.parameters.get("routing_strategy") + + if routing_strategy_param and routing_strategy_param.annotation: + # Extract Literal values using get_args + literal_values = get_args(routing_strategy_param.annotation) + if literal_values: + return list(literal_values) + + raise ValueError("Unable to extract routing strategies from Router class") + + +@router.get( + "/router/settings", + tags=["Router Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=RouterSettingsResponse, +) +async def get_router_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get router configuration and available settings. + + Returns: + - fields: List of all configurable router settings with their metadata (type, description, default, options) + The routing_strategy field includes available options extracted from the Router class + - current_values: Current values of router settings from config + """ + from litellm.proxy.proxy_server import llm_router, proxy_config + + try: + # Get available routing strategies dynamically from Router class + available_routing_strategies = _get_routing_strategies_from_router_class() + + # Get router settings fields from types file + router_fields = [field.model_copy(deep=True) for field in ROUTER_SETTINGS_FIELDS] + + # Populate routing_strategy field with available options and descriptions + for field in router_fields: + if field.field_name == "routing_strategy": + field.options = available_routing_strategies + break + + # Try to get router settings from config + config = await proxy_config.get_config() + router_settings_from_config = config.get("router_settings", {}) + + # Get current values from llm_router if initialized + current_values = {} + if llm_router is not None: + # Check all field names from the fields list + for field in router_fields: + if hasattr(llm_router, field.field_name): + value = getattr(llm_router, field.field_name) + current_values[field.field_name] = value + + # Merge with config values (config takes precedence) + current_values.update(router_settings_from_config) + + # Update field values with current values + for field in router_fields: + if field.field_name in current_values: + field.field_value = current_values[field.field_name] + + return RouterSettingsResponse( + fields=router_fields, + current_values=current_values, + routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Error fetching router settings: {str(e)}" + ) + raise + diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 78632628ae..ca34735879 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -263,6 +263,9 @@ from litellm.proxy.management_endpoints.cost_tracking_settings import ( from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, ) +from litellm.proxy.management_endpoints.router_settings_endpoints import ( + router as router_settings_router, +) from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) @@ -10040,6 +10043,7 @@ app.include_router(budget_management_router) app.include_router(model_management_router) app.include_router(tag_management_router) app.include_router(cost_tracking_settings_router) +app.include_router(router_settings_router) app.include_router(user_agent_analytics_router) app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) diff --git a/litellm/types/management_endpoints/__init__.py b/litellm/types/management_endpoints/__init__.py new file mode 100644 index 0000000000..90f7333edb --- /dev/null +++ b/litellm/types/management_endpoints/__init__.py @@ -0,0 +1,16 @@ +""" +Types for management endpoints +""" + +from .router_settings_endpoints import ( + ROUTER_SETTINGS_FIELDS, + ROUTING_STRATEGY_DESCRIPTIONS, + RouterSettingsField, +) + +__all__ = [ + "ROUTER_SETTINGS_FIELDS", + "ROUTING_STRATEGY_DESCRIPTIONS", + "RouterSettingsField", +] + diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py new file mode 100644 index 0000000000..9e3002ecf4 --- /dev/null +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -0,0 +1,197 @@ +""" +Types and field definitions for router settings management endpoints +""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + + +class RouterSettingsField(BaseModel): + field_name: str + field_type: str + field_value: Any + field_description: str + field_default: Any = None + options: Optional[List[str]] = None # For fields with predefined options/enum values + ui_field_name: str # User-friendly display name + link: Optional[str] = None # Documentation link for the field + + +# Routing strategy descriptions +ROUTING_STRATEGY_DESCRIPTIONS: Dict[str, str] = { + "simple-shuffle": "Randomly picks a deployment from the list. Simple and fast.", + "least-busy": "Routes to the deployment with the lowest number of ongoing requests.", + "latency-based-routing": "Routes to the deployment with the lowest latency over a sliding window.", + "cost-based-routing": "Routes to the deployment with the lowest cost per token.", + "usage-based-routing": "Routes to the deployment with the lowest TPM (Tokens Per Minute) usage. (deprecated)", + "usage-based-routing-v2": "Improved version of usage-based routing with better tracking.", +} + + +# Define all available router settings fields +ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [ + RouterSettingsField( + field_name="routing_strategy", + field_type="String", + field_value=None, + field_description="Routing strategy to use for load balancing across deployments", + field_default="simple-shuffle", + options=[], # Will be populated dynamically from Router class + ui_field_name="Routing Strategy", + ), + RouterSettingsField( + field_name="routing_strategy_args", + field_type="Dictionary", + field_value=None, + field_description="Arguments to pass to the routing strategy (e.g., ttl, lowest_latency_buffer for latency-based-routing)", + field_default={}, + ui_field_name="Routing Strategy Args", + ), + RouterSettingsField( + field_name="num_retries", + field_type="Integer", + field_value=None, + field_description="Number of retries for failed requests", + field_default=0, + ui_field_name="Number of Retries", + ), + RouterSettingsField( + field_name="timeout", + field_type="Float", + field_value=None, + field_description="Timeout for requests in seconds", + field_default=None, + ui_field_name="Timeout", + ), + RouterSettingsField( + field_name="stream_timeout", + field_type="Float", + field_value=None, + field_description="Timeout for streaming requests in seconds", + field_default=None, + ui_field_name="Stream Timeout", + ), + RouterSettingsField( + field_name="max_fallbacks", + field_type="Integer", + field_value=None, + field_description="Maximum number of fallbacks to try before exiting the call", + field_default=5, + ui_field_name="Max Fallbacks", + ), + RouterSettingsField( + field_name="fallbacks", + field_type="List", + field_value=None, + field_description="List of fallback model mappings", + field_default=[], + ui_field_name="Fallbacks", + ), + RouterSettingsField( + field_name="context_window_fallbacks", + field_type="List", + field_value=None, + field_description="List of fallback models for context window errors", + field_default=[], + ui_field_name="Context Window Fallbacks", + ), + RouterSettingsField( + field_name="content_policy_fallbacks", + field_type="List", + field_value=None, + field_description="List of fallback models for content policy errors", + field_default=[], + ui_field_name="Content Policy Fallbacks", + ), + RouterSettingsField( + field_name="allowed_fails", + field_type="Integer", + field_value=None, + field_description="Number of times a deployment can fail before being added to cooldown", + field_default=None, + ui_field_name="Allowed Fails", + ), + RouterSettingsField( + field_name="cooldown_time", + field_type="Float", + field_value=None, + field_description="Time in seconds to cooldown a deployment after failure", + field_default=None, + ui_field_name="Cooldown Time", + ), + RouterSettingsField( + field_name="retry_after", + field_type="Integer", + field_value=None, + field_description="Minimum time to wait before retrying a failed request in seconds", + field_default=0, + ui_field_name="Retry After", + ), + RouterSettingsField( + field_name="retry_policy", + field_type="Dictionary", + field_value=None, + field_description="Custom retry policy for different exception types", + field_default=None, + ui_field_name="Retry Policy", + ), + RouterSettingsField( + field_name="model_group_alias", + field_type="Dictionary", + field_value=None, + field_description="Aliases for model groups", + field_default={}, + ui_field_name="Model Group Alias", + ), + RouterSettingsField( + field_name="enable_pre_call_checks", + field_type="Boolean", + field_value=None, + field_description="Enable pre-call checks before routing requests", + field_default=False, + ui_field_name="Enable Pre-call Checks", + ), + RouterSettingsField( + field_name="default_litellm_params", + field_type="Dictionary", + field_value=None, + field_description="Default parameters for Router.chat.completion.create", + field_default=None, + ui_field_name="Default LiteLLM Params", + ), + RouterSettingsField( + field_name="set_verbose", + field_type="Boolean", + field_value=None, + field_description="Enable verbose logging for router", + field_default=False, + ui_field_name="Verbose Logging", + ), + RouterSettingsField( + field_name="default_max_parallel_requests", + field_type="Integer", + field_value=None, + field_description="Default maximum parallel requests across all deployments", + field_default=None, + ui_field_name="Max Parallel Requests", + ), + RouterSettingsField( + field_name="enable_tag_filtering", + field_type="Boolean", + field_value=None, + field_description="Enable tag-based routing to route requests based on tags", + field_default=False, + ui_field_name="Enable Tag Filtering", + link="https://docs.litellm.ai/docs/proxy/tag_routing", + ), + RouterSettingsField( + field_name="disable_cooldowns", + field_type="Boolean", + field_value=None, + field_description="Disable cooldown mechanism for failed deployments", + field_default=None, + ui_field_name="Disable Cooldowns", + ), +] + diff --git a/ui/litellm-dashboard/src/components/fallbacks.tsx b/ui/litellm-dashboard/src/components/fallbacks.tsx new file mode 100644 index 0000000000..1badc452ef --- /dev/null +++ b/ui/litellm-dashboard/src/components/fallbacks.tsx @@ -0,0 +1,169 @@ +import React, { useState, useEffect } from "react"; +import { + Table, + TableHead, + TableRow, + TableHeaderCell, + TableCell, + TableBody, + Button, + Icon, +} from "@tremor/react"; +import { + getCallbacksCall, + setCallbacksCall, +} from "./networking"; +import { TrashIcon } from "@heroicons/react/outline"; +import AddFallbacks from "./add_fallbacks"; +import openai from "openai"; +import NotificationsManager from "./molecules/notifications_manager"; + +interface FallbacksProps { + accessToken: string | null; + userRole: string | null; + userID: string | null; + modelData: any; +} + +async function testFallbackModelResponse(selectedModel: string, accessToken: string) { + const isLocal = process.env.NODE_ENV === "development"; + if (isLocal != true) { + console.log = function () {}; + } + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; + const client = new openai.OpenAI({ + apiKey: accessToken, + baseURL: proxyBaseUrl, + dangerouslyAllowBrowser: true, + }); + + try { + const response = await client.chat.completions.create({ + model: selectedModel, + messages: [ + { + role: "user", + content: "Hi, this is a test message", + }, + ], + // @ts-ignore + mock_testing_fallbacks: true, + }); + + NotificationsManager.success( + + Test model={selectedModel}, received model= + {response.model}. See{" "} + window.open("https://docs.litellm.ai/docs/proxy/reliability", "_blank")} + style={{ textDecoration: "underline", color: "blue" }} + > + curl + + , + ); + } catch (error) { + NotificationsManager.fromBackend( + `Error occurred while generating model response. Please try again. Error: ${error}`, + ); + } +} + +const Fallbacks: React.FC = ({ accessToken, userRole, userID, modelData }) => { + const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({}); + + useEffect(() => { + if (!accessToken || !userRole || !userID) { + return; + } + getCallbacksCall(accessToken, userID, userRole).then((data) => { + console.log("callbacks", data); + let router_settings = data.router_settings; + if ("model_group_retry_policy" in router_settings) { + delete router_settings["model_group_retry_policy"]; + } + setRouterSettings(router_settings); + }); + }, [accessToken, userRole, userID]); + + const deleteFallbacks = async (key: string) => { + if (!accessToken) { + return; + } + + console.log(`received key: ${key}`); + console.log(`routerSettings['fallbacks']: ${routerSettings["fallbacks"]}`); + + const updatedFallbacks = routerSettings["fallbacks"] + .map((dict: { [key: string]: any }) => { + if (key in dict) { + delete dict[key]; + } + return dict; + }) + .filter((dict: { [key: string]: any }) => Object.keys(dict).length > 0); + + const updatedSettings = { + ...routerSettings, + fallbacks: updatedFallbacks, + }; + + const payload = { + router_settings: updatedSettings, + }; + + try { + await setCallbacksCall(accessToken, payload); + setRouterSettings(updatedSettings); + NotificationsManager.success("Router settings updated successfully"); + } catch (error) { + NotificationsManager.fromBackend("Failed to update router settings: " + error); + } + }; + + if (!accessToken) { + return null; + } + + return ( + <> + + + + Model Name + Fallbacks + + + + + {routerSettings["fallbacks"] && + routerSettings["fallbacks"].map((item: object, index: number) => + Object.entries(item).map(([key, value]) => ( + + {key} + {Array.isArray(value) ? value.join(", ") : value} + + + + + deleteFallbacks(key)} /> + + + )), + )} + +
+ data.model_name) : []} + accessToken={accessToken} + routerSettings={routerSettings} + setRouterSettings={setRouterSettings} + /> + + ); +}; + +export default Fallbacks; + diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index f7a6781d36..9c49bcf604 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -10,20 +10,11 @@ import { TableCell, TableBody, Text, - Grid, Button, - TextInput, - Select as Select2, - SelectItem, - Col, - Accordion, - AccordionBody, - AccordionHeader, + Icon, } from "@tremor/react"; -import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; +import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; import { - getCallbacksCall, - setCallbacksCall, getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting, @@ -31,9 +22,8 @@ import { import { Form, InputNumber } from "antd"; import { TrashIcon, CheckCircleIcon } from "@heroicons/react/outline"; -import AddFallbacks from "./add_fallbacks"; -import openai from "openai"; -import NotificationsManager from "./molecules/notifications_manager"; +import RouterSettings from "./router_settings"; +import Fallbacks from "./fallbacks"; interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; @@ -41,64 +31,6 @@ interface GeneralSettingsPageProps { modelData: any; } -async function testFallbackModelResponse(selectedModel: string, accessToken: string) { - // base url should be the current base_url - const isLocal = process.env.NODE_ENV === "development"; - if (isLocal != true) { - console.log = function () {}; - } - console.log("isLocal:", isLocal); - const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; - const client = new openai.OpenAI({ - apiKey: accessToken, // Replace with your OpenAI API key - baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL - dangerouslyAllowBrowser: true, // using a temporary litellm proxy key - }); - - try { - const response = await client.chat.completions.create({ - model: selectedModel, - messages: [ - { - role: "user", - content: "Hi, this is a test message", - }, - ], - // @ts-ignore - mock_testing_fallbacks: true, - }); - - NotificationsManager.success( - - Test model={selectedModel}, received model= - {response.model}. See{" "} - window.open("https://docs.litellm.ai/docs/proxy/reliability", "_blank")} - style={{ textDecoration: "underline", color: "blue" }} - > - curl - - , - ); - } catch (error) { - NotificationsManager.fromBackend( - `Error occurred while generating model response. Please try again. Error: ${error}`, - ); - } -} - -interface AccordionHeroProps { - selectedStrategy: string | null; - strategyArgs: routingStrategyArgs; - paramExplanation: { [key: string]: string }; -} - -interface routingStrategyArgs { - ttl?: number; - lowest_latency_buffer?: number; -} - interface generalSettingsItem { field_name: string; field_type: string; @@ -107,152 +39,18 @@ interface generalSettingsItem { stored_in_db: boolean | null; } -const defaultLowestLatencyArgs: routingStrategyArgs = { - ttl: 3600, - lowest_latency_buffer: 0, -}; - -export const AccordionHero: React.FC = ({ selectedStrategy, strategyArgs, paramExplanation }) => ( - - - Routing Strategy Specific Args - - - {selectedStrategy == "latency-based-routing" ? ( - - - - - Setting - Value - - - - {Object.entries(strategyArgs).map(([param, value]) => ( - - - {param} -

- {paramExplanation[param]} -

- - - - - - ))} - -
- - ) : ( - No specific settings - )} - - -); - const GeneralSettings: React.FC = ({ accessToken, userRole, userID, modelData }) => { - const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({}); - const [generalSettingsDict, setGeneralSettingsDict] = useState<{ - [key: string]: any; - }>({}); const [generalSettings, setGeneralSettings] = useState([]); - const [isModalVisible, setIsModalVisible] = useState(false); - const [form] = Form.useForm(); - const [selectedCallback, setSelectedCallback] = useState(null); - const [selectedStrategy, setSelectedStrategy] = useState(null); - const [strategySettings, setStrategySettings] = useState(null); - - let paramExplanation: { [key: string]: string } = { - routing_strategy_args: "(dict) Arguments to pass to the routing strategy", - routing_strategy: "(string) Routing strategy to use", - allowed_fails: "(int) Number of times a deployment can fail before being added to cooldown", - cooldown_time: "(int) time in seconds to cooldown a deployment after failure", - num_retries: "(int) Number of retries for failed requests. Defaults to 0.", - timeout: "(float) Timeout for requests. Defaults to None.", - retry_after: "(int) Minimum time to wait before retrying a failed request", - ttl: "(int) Sliding window to look back over when calculating the average latency of a deployment. Default - 1 hour (in seconds).", - lowest_latency_buffer: - "(float) Shuffle between deployments within this % of the lowest latency. Default - 0 (i.e. always pick lowest latency).", - }; useEffect(() => { - if (!accessToken || !userRole || !userID) { + if (!accessToken) { return; } - getCallbacksCall(accessToken, userID, userRole).then((data) => { - console.log("callbacks", data); - let router_settings = data.router_settings; - // remove "model_group_retry_policy" from general_settings if exists - if ("model_group_retry_policy" in router_settings) { - delete router_settings["model_group_retry_policy"]; - } - setRouterSettings(router_settings); - }); getGeneralSettingsCall(accessToken).then((data) => { let general_settings = data; setGeneralSettings(general_settings); }); - }, [accessToken, userRole, userID]); - - const handleAddCallback = () => { - console.log("Add callback clicked"); - setIsModalVisible(true); - }; - - const handleCancel = () => { - setIsModalVisible(false); - form.resetFields(); - setSelectedCallback(null); - }; - - const deleteFallbacks = async (key: string) => { - /** - * pop the key from the Object, if it exists - */ - if (!accessToken) { - return; - } - - console.log(`received key: ${key}`); - console.log(`routerSettings['fallbacks']: ${routerSettings["fallbacks"]}`); - - const updatedFallbacks = routerSettings["fallbacks"] - .map((dict: { [key: string]: any }) => { - if (key in dict) { - delete dict[key]; - } - return dict; - }) - .filter((dict: { [key: string]: any }) => Object.keys(dict).length > 0); - - const updatedSettings = { - ...routerSettings, - fallbacks: updatedFallbacks, - }; - - const payload = { - router_settings: updatedSettings, - }; - - try { - await setCallbacksCall(accessToken, payload); - setRouterSettings(updatedSettings); - NotificationsManager.success("Router settings updated successfully"); - } catch (error) { - NotificationsManager.fromBackend("Failed to update router settings: " + error); - } - }; + }, [accessToken]); const handleInputChange = (fieldName: string, newValue: any) => { // Update the value in the state @@ -303,212 +101,33 @@ const GeneralSettings: React.FC = ({ accessToken, user } }; - const handleSaveChanges = (router_settings: any) => { - if (!accessToken) { - return; - } - - console.log("router_settings", router_settings); - - const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); - const jsonKeys = new Set(["model_group_alias", "retry_policy"]); - - const parseInputValue = (key: string, raw: string | undefined, fallback: unknown) => { - if (raw === undefined) return fallback; - - const v = raw.trim(); - - if (v.toLowerCase() === "null") return null; - - if (numberKeys.has(key)) { - const n = Number(v); - return Number.isNaN(n) ? fallback : n; - } - - if (jsonKeys.has(key)) { - if (v === "") return null; - try { - return JSON.parse(v); - } catch { - return fallback; - } - } - - if (v.toLowerCase() === "true") return true; - if (v.toLowerCase() === "false") return false; - - return v; - }; - - const updatedVariables = Object.fromEntries( - Object.entries(router_settings) - .map(([key, value]) => { - if (key !== "routing_strategy_args" && key !== "routing_strategy") { - const inputEl = document.querySelector(`input[name="${key}"]`) as HTMLInputElement | null; - const parsed = parseInputValue(key, inputEl?.value, value); - return [key, parsed]; - } else if (key === "routing_strategy") { - return [key, selectedStrategy]; - } else if (key === "routing_strategy_args" && selectedStrategy === "latency-based-routing") { - let setRoutingStrategyArgs: routingStrategyArgs = {}; - - const lowestLatencyBufferElement = document.querySelector( - `input[name="lowest_latency_buffer"]`, - ) as HTMLInputElement; - const ttlElement = document.querySelector(`input[name="ttl"]`) as HTMLInputElement; - - if (lowestLatencyBufferElement?.value) { - setRoutingStrategyArgs["lowest_latency_buffer"] = Number(lowestLatencyBufferElement.value); - } - - if (ttlElement?.value) { - setRoutingStrategyArgs["ttl"] = Number(ttlElement.value); - } - - console.log(`setRoutingStrategyArgs: ${setRoutingStrategyArgs}`); - return ["routing_strategy_args", setRoutingStrategyArgs]; - } - return null; - }) - .filter((entry) => entry !== null && entry !== undefined) as Iterable<[string, unknown]>, - ); - console.log("updatedVariables", updatedVariables); - - const payload = { - router_settings: updatedVariables, - }; - - try { - setCallbacksCall(accessToken, payload); - } catch (error) { - NotificationsManager.fromBackend("Failed to update router settings: " + error); - } - - NotificationsManager.success("router settings updated successfully"); - }; - if (!accessToken) { return null; } return ( -
- - +
+ + Loadbalancing Fallbacks General - + - - Router Settings - - - - - Setting - Value - - - - {Object.entries(routerSettings) - .filter( - ([param, value]) => - param != "fallbacks" && - param != "context_window_fallbacks" && - param != "routing_strategy_args", - ) - .map(([param, value]) => ( - - - {param} -

- {paramExplanation[param]} -

- - - {param == "routing_strategy" ? ( - - usage-based-routing - latency-based-routing - simple-shuffle - - ) : ( - - )} - - - ))} - -
- 0 - ? routerSettings["routing_strategy_args"] - : defaultLowestLatencyArgs // default value when keys length is 0 - } - paramExplanation={paramExplanation} - /> - - - - - + - - - - Model Name - Fallbacks - - - - - {routerSettings["fallbacks"] && - routerSettings["fallbacks"].map((item: object, index: number) => - Object.entries(item).map(([key, value]) => ( - - {key} - {Array.isArray(value) ? value.join(", ") : value} - - - - - deleteFallbacks(key)} /> - - - )), - )} - -
- data.model_name) : []} + diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 41de54ca67..b7489e7efc 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4442,6 +4442,35 @@ export const getGeneralSettingsCall = async (accessToken: string) => { } }; +export const getRouterSettingsCall = async (accessToken: string) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/router/settings` + : `/router/settings`; + + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to get router settings:", error); + throw error; + } +}; + export const getPassThroughEndpointsCall = async (accessToken: string, teamId?: string | null) => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`; diff --git a/ui/litellm-dashboard/src/components/router_settings/LatencyBasedConfiguration.tsx b/ui/litellm-dashboard/src/components/router_settings/LatencyBasedConfiguration.tsx new file mode 100644 index 0000000000..9c776c94a0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/LatencyBasedConfiguration.tsx @@ -0,0 +1,62 @@ +import React from "react"; +import { TextInput } from "@tremor/react"; + +interface routingStrategyArgs { + ttl?: number; + lowest_latency_buffer?: number; +} + +const defaultLowestLatencyArgs: routingStrategyArgs = { + ttl: 3600, + lowest_latency_buffer: 0, +}; + +interface LatencyBasedConfigurationProps { + routingStrategyArgs: { [key: string]: any }; +} + +const LatencyBasedConfiguration: React.FC = ({ + routingStrategyArgs, +}) => { + const paramExplanation: { [key: string]: string } = { + ttl: "Sliding window to look back over when calculating the average latency of a deployment. Default - 1 hour (in seconds).", + lowest_latency_buffer: + "Shuffle between deployments within this % of the lowest latency. Default - 0 (i.e. always pick lowest latency).", + }; + + return ( + <> +
+
+

Latency-Based Configuration

+

Fine-tune latency-based routing behavior

+
+ +
+ {Object.entries(routingStrategyArgs || defaultLowestLatencyArgs).map(([param, value]) => ( +
+ +
+ ))} +
+
+ +
+ + ); +}; + +export default LatencyBasedConfiguration; + diff --git a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx new file mode 100644 index 0000000000..6601950fde --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import { TextInput } from "@tremor/react"; + +interface ReliabilityRetriesSectionProps { + routerSettings: { [key: string]: any }; + routerFieldsMetadata: { [key: string]: any }; +} + +const ReliabilityRetriesSection: React.FC = ({ + routerSettings, + routerFieldsMetadata, +}) => { + return ( +
+
+

Reliability & Retries

+

Configure retry logic and failure handling

+
+ +
+ {Object.entries(routerSettings) + .filter( + ([param, value]) => + param != "fallbacks" && + param != "context_window_fallbacks" && + param != "routing_strategy_args" && + param != "routing_strategy" && + param != "enable_tag_filtering", + ) + .map(([param, value]) => ( +
+ +
+ ))} +
+
+ ); +}; + +export default ReliabilityRetriesSection; + diff --git a/ui/litellm-dashboard/src/components/router_settings/RoutingStrategySelector.tsx b/ui/litellm-dashboard/src/components/router_settings/RoutingStrategySelector.tsx new file mode 100644 index 0000000000..c64fb410e7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/RoutingStrategySelector.tsx @@ -0,0 +1,55 @@ +import React from "react"; +import { Select } from "antd"; + +interface RoutingStrategySelectorProps { + selectedStrategy: string | null; + availableStrategies: string[]; + routingStrategyDescriptions: { [key: string]: string }; + routerFieldsMetadata: { [key: string]: any }; + onStrategyChange: (strategy: string) => void; +} + +const RoutingStrategySelector: React.FC = ({ + selectedStrategy, + availableStrategies, + routingStrategyDescriptions, + routerFieldsMetadata, + onStrategyChange, +}) => { + return ( +
+
+ +

+ {routerFieldsMetadata["routing_strategy"]?.field_description || ""} +

+
+
+ +
+
+ ); +}; + +export default RoutingStrategySelector; + diff --git a/ui/litellm-dashboard/src/components/router_settings/TagFilteringToggle.tsx b/ui/litellm-dashboard/src/components/router_settings/TagFilteringToggle.tsx new file mode 100644 index 0000000000..2b53a48c3a --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/TagFilteringToggle.tsx @@ -0,0 +1,50 @@ +import React from "react"; +import { Switch } from "@tremor/react"; + +interface TagFilteringToggleProps { + enabled: boolean; + routerFieldsMetadata: { [key: string]: any }; + onToggle: (enabled: boolean) => void; +} + +const TagFilteringToggle: React.FC = ({ + enabled, + routerFieldsMetadata, + onToggle, +}) => { + return ( +
+
+
+ +

+ {routerFieldsMetadata["enable_tag_filtering"]?.field_description || ""} + {routerFieldsMetadata["enable_tag_filtering"]?.link && ( + <> + {" "} + + Learn more + + + )} +

+
+ +
+
+ ); +}; + +export default TagFilteringToggle; + diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx new file mode 100644 index 0000000000..56b87cff8b --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx @@ -0,0 +1,255 @@ +import React, { useState, useEffect } from "react"; +import { + Button, + TextInput, +} from "@tremor/react"; +import { + getCallbacksCall, + setCallbacksCall, + getRouterSettingsCall, +} from "../networking"; +import NotificationsManager from "../molecules/notifications_manager"; +import RoutingStrategySelector from "./RoutingStrategySelector"; +import TagFilteringToggle from "./TagFilteringToggle"; +import LatencyBasedConfiguration from "./LatencyBasedConfiguration"; +import ReliabilityRetriesSection from "./ReliabilityRetriesSection"; + +interface RouterSettingsProps { + accessToken: string | null; + userRole: string | null; + userID: string | null; + modelData: any; +} + +interface routingStrategyArgs { + ttl?: number; + lowest_latency_buffer?: number; +} + +const RouterSettings: React.FC = ({ accessToken, userRole, userID, modelData }) => { + const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({}); + const [selectedStrategy, setSelectedStrategy] = useState(null); + const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState([]); + const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({}); + const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({}); + const [enableTagFiltering, setEnableTagFiltering] = useState(false); + + useEffect(() => { + if (!accessToken || !userRole || !userID) { + return; + } + getCallbacksCall(accessToken, userID, userRole).then((data) => { + console.log("callbacks", data); + let router_settings = data.router_settings; + if ("model_group_retry_policy" in router_settings) { + delete router_settings["model_group_retry_policy"]; + } + setRouterSettings(router_settings); + // Set initial selected strategy + if (router_settings.routing_strategy) { + setSelectedStrategy(router_settings.routing_strategy); + } + }); + getRouterSettingsCall(accessToken).then((data) => { + console.log("router settings from API", data); + if (data.fields) { + // Build metadata map for easy lookup + const fieldsMap: { [key: string]: any } = {}; + data.fields.forEach((field: any) => { + fieldsMap[field.field_name] = { + ui_field_name: field.ui_field_name, + field_description: field.field_description, + options: field.options, + link: field.link, + }; + }); + setRouterFieldsMetadata(fieldsMap); + + // Extract routing strategies from the routing_strategy field's options + const routingStrategyField = data.fields.find( + (field: any) => field.field_name === "routing_strategy" + ); + if (routingStrategyField?.options) { + setAvailableRoutingStrategies(routingStrategyField.options); + } + + // Store routing strategy descriptions + if (data.routing_strategy_descriptions) { + setRoutingStrategyDescriptions(data.routing_strategy_descriptions); + } + + // Set enable_tag_filtering value + const tagFilteringField = data.fields.find( + (field: any) => field.field_name === "enable_tag_filtering" + ); + if (tagFilteringField?.field_value !== null && tagFilteringField?.field_value !== undefined) { + setEnableTagFiltering(tagFilteringField.field_value); + } + } + }); + }, [accessToken, userRole, userID]); + + const handleSaveChanges = (router_settings: any) => { + if (!accessToken) { + return; + } + + console.log("router_settings", router_settings); + + const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); + const jsonKeys = new Set(["model_group_alias", "retry_policy"]); + + const parseInputValue = (key: string, raw: string | undefined, fallback: unknown) => { + if (raw === undefined) return fallback; + + const v = raw.trim(); + + if (v.toLowerCase() === "null") return null; + + if (numberKeys.has(key)) { + const n = Number(v); + return Number.isNaN(n) ? fallback : n; + } + + if (jsonKeys.has(key)) { + if (v === "") return null; + try { + return JSON.parse(v); + } catch { + return fallback; + } + } + + if (v.toLowerCase() === "true") return true; + if (v.toLowerCase() === "false") return false; + + return v; + }; + + // Add enable_tag_filtering to router_settings before processing + const settingsToUpdate = { + ...router_settings, + enable_tag_filtering: enableTagFiltering, + }; + + const updatedVariables = Object.fromEntries( + Object.entries(settingsToUpdate) + .map(([key, value]) => { + if (key !== "routing_strategy_args" && key !== "routing_strategy" && key !== "enable_tag_filtering") { + const inputEl = document.querySelector(`input[name="${key}"]`) as HTMLInputElement | null; + const parsed = parseInputValue(key, inputEl?.value, value); + return [key, parsed]; + } else if (key === "routing_strategy") { + return [key, selectedStrategy]; + } else if (key === "enable_tag_filtering") { + return [key, enableTagFiltering]; + } else if (key === "routing_strategy_args" && selectedStrategy === "latency-based-routing") { + let setRoutingStrategyArgs: routingStrategyArgs = {}; + + const lowestLatencyBufferElement = document.querySelector( + `input[name="lowest_latency_buffer"]`, + ) as HTMLInputElement; + const ttlElement = document.querySelector(`input[name="ttl"]`) as HTMLInputElement; + + if (lowestLatencyBufferElement?.value) { + setRoutingStrategyArgs["lowest_latency_buffer"] = Number(lowestLatencyBufferElement.value); + } + + if (ttlElement?.value) { + setRoutingStrategyArgs["ttl"] = Number(ttlElement.value); + } + + console.log(`setRoutingStrategyArgs: ${setRoutingStrategyArgs}`); + return ["routing_strategy_args", setRoutingStrategyArgs]; + } + return null; + }) + .filter((entry) => entry !== null && entry !== undefined) as Iterable<[string, unknown]>, + ); + console.log("updatedVariables", updatedVariables); + + const payload = { + router_settings: updatedVariables, + }; + + try { + setCallbacksCall(accessToken, payload); + } catch (error) { + NotificationsManager.fromBackend("Failed to update router settings: " + error); + } + + NotificationsManager.success("router settings updated successfully"); + }; + + if (!accessToken) { + return null; + } + + return ( +
+ {/* Routing Settings Section */} +
+
+

Routing Settings

+

Configure how requests are routed to deployments

+
+ + {/* Routing Strategy */} + {routerSettings.routing_strategy && ( + + )} + + {/* Tag Filtering */} + +
+ + {/* Divider */} +
+ + {/* Strategy-Specific Args - Show immediately after strategy if latency-based */} + {selectedStrategy === "latency-based-routing" && ( + + )} + + {/* Other Settings */} + + + {/* Actions - Sticky at bottom */} +
+ + +
+
+ ); +}; + +export default RouterSettings; +