mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-18 02:17:35 +00:00
Merge pull request #27131 from BerriAI/litellm_fix/routing-groups-ui
feat: routing groups ui
This commit is contained in:
Yassin Kortam
@@ -104,11 +104,16 @@ async def get_router_settings(
|
||||
config = await proxy_config.get_config()
|
||||
router_settings_from_config = config.get("router_settings", {})
|
||||
|
||||
# Get current values from llm_router if initialized
|
||||
current_values = {}
|
||||
current_values: Dict[str, Any] = {}
|
||||
if llm_router is not None:
|
||||
# Check all field names from the fields list
|
||||
# Router exposes routing groups as private `_routing_groups`; the
|
||||
# generic `hasattr` loop below would miss them.
|
||||
current_values["routing_groups"] = [
|
||||
group.model_dump() for group in llm_router._routing_groups.values()
|
||||
]
|
||||
for field in router_fields:
|
||||
if field.field_name == "routing_groups":
|
||||
continue
|
||||
if hasattr(llm_router, field.field_name):
|
||||
value = getattr(llm_router, field.field_name)
|
||||
current_values[field.field_name] = value
|
||||
|
||||
@@ -1052,11 +1052,17 @@ class Router:
|
||||
strategy = self._normalize_strategy(self.routing_strategy)
|
||||
attr = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "")
|
||||
selector = getattr(self, attr, None) if attr is not None else None
|
||||
verbose_router_logger.debug(
|
||||
"routing_group=default model=%s strategy=%s", model, strategy
|
||||
)
|
||||
return strategy, selector
|
||||
|
||||
group = self._routing_groups[group_name]
|
||||
strategy = self._normalize_strategy(group.routing_strategy)
|
||||
selector = self._group_selectors.get(group_name, {}).get(strategy or "")
|
||||
verbose_router_logger.debug(
|
||||
"routing_group=%s model=%s strategy=%s", group_name, model, strategy
|
||||
)
|
||||
return strategy, selector
|
||||
|
||||
async def _select_deployment_async(
|
||||
|
||||
@@ -112,6 +112,14 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [
|
||||
field_default={},
|
||||
ui_field_name="Routing Strategy Args",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="routing_groups",
|
||||
field_type="List",
|
||||
field_value=None,
|
||||
field_description="Named subsets of model_names that share a routing strategy. Models not claimed by an explicit group fall through to the top-level routing_strategy.",
|
||||
field_default=[],
|
||||
ui_field_name="Routing Groups",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="num_retries",
|
||||
field_type="Integer",
|
||||
|
||||
@@ -18,6 +18,7 @@ from litellm import Router
|
||||
|
||||
# this tests debug logs from litellm router and litellm proxy server
|
||||
from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger
|
||||
from litellm.llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
|
||||
|
||||
# this tests debug logs from litellm router and litellm proxy server
|
||||
@@ -74,6 +75,9 @@ def test_async_fallbacks(caplog):
|
||||
pytest.fail(f"An exception occurred: {e}")
|
||||
finally:
|
||||
router.reset()
|
||||
# Close cached aiohttp/httpx clients before the event loop ends
|
||||
# to prevent "Unclosed client session" / "Unclosed connector" warnings.
|
||||
await close_litellm_async_clients()
|
||||
|
||||
asyncio.run(_make_request())
|
||||
captured_logs = [rec.message for rec in caplog.records]
|
||||
|
||||
@@ -13,7 +13,13 @@ from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../.."))
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.router_settings_endpoints import (
|
||||
get_router_settings,
|
||||
)
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.router import Router
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -71,3 +77,48 @@ class TestRouterSettingsEndpoints:
|
||||
assert "options" in routing_strategy_field
|
||||
assert isinstance(routing_strategy_field["options"], list)
|
||||
assert len(routing_strategy_field["options"]) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_settings_includes_routing_groups_from_live_router(
|
||||
self, monkeypatch
|
||||
):
|
||||
"""GET /router/settings returns routing_groups from the live router."""
|
||||
groups = [
|
||||
{
|
||||
"group_name": "test-group",
|
||||
"models": ["latency-model"],
|
||||
"routing_strategy": "latency-based-routing",
|
||||
"routing_strategy_args": {},
|
||||
}
|
||||
]
|
||||
llm_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "latency-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-x",
|
||||
},
|
||||
}
|
||||
],
|
||||
routing_groups=groups,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", llm_router)
|
||||
|
||||
async def fake_get_config(self, config_file_path=None):
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
|
||||
)
|
||||
|
||||
admin_user = UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
|
||||
)
|
||||
response = await get_router_settings(user_api_key_dict=admin_user)
|
||||
|
||||
assert response.current_values.get("routing_groups") == groups
|
||||
|
||||
rg_field = next(f for f in response.fields if f.field_name == "routing_groups")
|
||||
assert rg_field.field_value == groups
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"use client";
|
||||
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { useMutation, useQuery, useQueryClient, UseMutationResult, UseQueryResult } from "@tanstack/react-query";
|
||||
import { getRouterSettingsCall, setCallbacksCall } from "@/components/networking";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
import type { RoutingGroup } from "@/components/routing_groups/types";
|
||||
|
||||
const routingGroupsKeys = createQueryKeys("routingGroups");
|
||||
|
||||
interface RoutingGroupsQueryData {
|
||||
routingGroups: RoutingGroup[];
|
||||
routingStrategy: string | null;
|
||||
availableStrategies: string[];
|
||||
}
|
||||
|
||||
const fetchRoutingGroups = async (accessToken: string): Promise<RoutingGroupsQueryData> => {
|
||||
const data = await getRouterSettingsCall(accessToken);
|
||||
const currentValues = data?.current_values ?? {};
|
||||
const fields = Array.isArray(data?.fields) ? data.fields : [];
|
||||
const routingStrategyField = fields.find((f: any) => f?.field_name === "routing_strategy");
|
||||
|
||||
return {
|
||||
routingGroups: Array.isArray(currentValues.routing_groups) ? currentValues.routing_groups : [],
|
||||
routingStrategy: currentValues.routing_strategy ?? null,
|
||||
availableStrategies: Array.isArray(routingStrategyField?.options) ? routingStrategyField.options : [],
|
||||
};
|
||||
};
|
||||
|
||||
export const useRoutingGroups = (): UseQueryResult<RoutingGroupsQueryData> => {
|
||||
const { accessToken, userId, userRole } = useAuthorized();
|
||||
return useQuery<RoutingGroupsQueryData>({
|
||||
queryKey: routingGroupsKeys.lists(),
|
||||
queryFn: () => fetchRoutingGroups(accessToken!),
|
||||
enabled: Boolean(accessToken && userId && userRole),
|
||||
});
|
||||
};
|
||||
|
||||
export const useSaveRoutingGroups = (): UseMutationResult<unknown, Error, RoutingGroup[]> => {
|
||||
const { accessToken } = useAuthorized();
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (routingGroups: RoutingGroup[]) =>
|
||||
setCallbacksCall(accessToken!, {
|
||||
router_settings: { routing_groups: routingGroups },
|
||||
}),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: routingGroupsKeys.lists() });
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -24,6 +24,7 @@ import { TrashIcon, CheckCircleIcon } from "@heroicons/react/outline";
|
||||
|
||||
import RouterSettings from "./router_settings";
|
||||
import Fallbacks from "./Settings/RouterSettings/Fallbacks/Fallbacks";
|
||||
import RoutingGroups from "./routing_groups";
|
||||
interface GeneralSettingsPageProps {
|
||||
accessToken: string | null;
|
||||
userRole: string | null;
|
||||
@@ -110,8 +111,9 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
||||
<TabGroup className="h-[75vh] w-full">
|
||||
<TabList variant="line" defaultValue="1" className="px-8 pt-4">
|
||||
<Tab value="1">Loadbalancing</Tab>
|
||||
<Tab value="2">Fallbacks</Tab>
|
||||
<Tab value="3">General</Tab>
|
||||
<Tab value="2">Routing Groups</Tab>
|
||||
<Tab value="3">Fallbacks</Tab>
|
||||
<Tab value="4">General</Tab>
|
||||
</TabList>
|
||||
<TabPanels className="px-8 py-6">
|
||||
<TabPanel>
|
||||
@@ -122,6 +124,9 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
||||
modelData={modelData}
|
||||
/>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<RoutingGroups />
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<Fallbacks
|
||||
accessToken={accessToken}
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"use client";
|
||||
|
||||
import React, { useMemo } from "react";
|
||||
import { Form, Input, Modal, Select, Space, Typography } from "antd";
|
||||
import type { RoutingGroup, RoutingStrategy } from "./types";
|
||||
|
||||
const { Text, Paragraph } = Typography;
|
||||
|
||||
interface RoutingGroupModalProps {
|
||||
open: boolean;
|
||||
mode: "create" | "edit";
|
||||
initialValue: RoutingGroup | null;
|
||||
availableStrategies: string[];
|
||||
strategyDescriptions: Record<string, string>;
|
||||
modelOptions: string[];
|
||||
existingGroupNames: string[];
|
||||
onClose: () => void;
|
||||
onSubmit: (group: RoutingGroup) => Promise<void> | void;
|
||||
saving?: boolean;
|
||||
}
|
||||
|
||||
interface FormValues {
|
||||
group_name: string;
|
||||
models: string[];
|
||||
routing_strategy: RoutingStrategy | string;
|
||||
routing_strategy_args?: string;
|
||||
}
|
||||
|
||||
const STRATEGIES_WITH_ARGS = new Set<string>(["latency-based-routing", "usage-based-routing"]);
|
||||
|
||||
const GROUP_NAME_PATTERN = /^[A-Za-z0-9._-]+$/;
|
||||
const GROUP_NAME_MAX_LENGTH = 64;
|
||||
|
||||
const RoutingGroupModal: React.FC<RoutingGroupModalProps> = ({
|
||||
open,
|
||||
mode,
|
||||
initialValue,
|
||||
availableStrategies,
|
||||
strategyDescriptions,
|
||||
modelOptions,
|
||||
existingGroupNames,
|
||||
onClose,
|
||||
onSubmit,
|
||||
saving,
|
||||
}) => {
|
||||
const [form] = Form.useForm<FormValues>();
|
||||
const selectedStrategy = Form.useWatch("routing_strategy", form);
|
||||
|
||||
const initialValues: FormValues = {
|
||||
group_name: initialValue?.group_name ?? "",
|
||||
models: initialValue?.models ?? [],
|
||||
routing_strategy: initialValue?.routing_strategy ?? availableStrategies[0] ?? "simple-shuffle",
|
||||
routing_strategy_args: initialValue?.routing_strategy_args
|
||||
? JSON.stringify(initialValue.routing_strategy_args, null, 2)
|
||||
: "",
|
||||
};
|
||||
|
||||
const reservedNames = useMemo(() => {
|
||||
const others = existingGroupNames.filter((n) => n !== initialValue?.group_name);
|
||||
return new Set(others.map((n) => n.toLowerCase()));
|
||||
}, [existingGroupNames, initialValue]);
|
||||
|
||||
const handleSubmit = async () => {
|
||||
const values = await form.validateFields();
|
||||
const strategySupportsArgs = STRATEGIES_WITH_ARGS.has(String(values.routing_strategy));
|
||||
let parsedArgs: Record<string, unknown> | null = null;
|
||||
if (strategySupportsArgs && values.routing_strategy_args && values.routing_strategy_args.trim()) {
|
||||
try {
|
||||
parsedArgs = JSON.parse(values.routing_strategy_args);
|
||||
} catch {
|
||||
form.setFields([
|
||||
{
|
||||
name: "routing_strategy_args",
|
||||
errors: ["Must be valid JSON"],
|
||||
},
|
||||
]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await onSubmit({
|
||||
group_name: values.group_name.trim(),
|
||||
models: values.models,
|
||||
routing_strategy: values.routing_strategy,
|
||||
routing_strategy_args: parsedArgs,
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={mode === "create" ? "Create Routing Group" : `Edit ${initialValue?.group_name ?? ""}`}
|
||||
open={open}
|
||||
onCancel={onClose}
|
||||
onOk={handleSubmit}
|
||||
okText={mode === "create" ? "Create Group" : "Save Changes"}
|
||||
cancelText="Cancel"
|
||||
confirmLoading={saving}
|
||||
destroyOnClose
|
||||
width={560}
|
||||
>
|
||||
<Form<FormValues>
|
||||
key={mode === "edit" ? `edit-${initialValue?.group_name ?? ""}` : "create"}
|
||||
form={form}
|
||||
layout="vertical"
|
||||
preserve={false}
|
||||
initialValues={initialValues}
|
||||
>
|
||||
<Form.Item
|
||||
label="Group Name"
|
||||
name="group_name"
|
||||
rules={[
|
||||
{ required: true, message: "Group name is required" },
|
||||
{ max: GROUP_NAME_MAX_LENGTH, message: `Must be ${GROUP_NAME_MAX_LENGTH} characters or fewer` },
|
||||
{
|
||||
pattern: GROUP_NAME_PATTERN,
|
||||
message: "Only letters, numbers, dot, underscore, and dash are allowed",
|
||||
},
|
||||
{
|
||||
validator: (_, value: string) => {
|
||||
if (!value) return Promise.resolve();
|
||||
if (reservedNames.has(value.trim().toLowerCase())) {
|
||||
return Promise.reject(new Error("A group with this name already exists"));
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
},
|
||||
]}
|
||||
extra="Use this name as the model in API calls — LiteLLM routes the request to one of the group's models."
|
||||
>
|
||||
<Input placeholder="fast-chat" disabled={mode === "edit"} />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="Models"
|
||||
name="models"
|
||||
rules={[{ required: true, message: "Select at least one model" }]}
|
||||
extra="Models from your model list that this group routes between."
|
||||
>
|
||||
<Select
|
||||
mode="multiple"
|
||||
allowClear
|
||||
placeholder="Select models"
|
||||
options={modelOptions.map((m) => ({ label: m, value: m }))}
|
||||
optionFilterProp="label"
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="Routing Strategy"
|
||||
name="routing_strategy"
|
||||
rules={[{ required: true, message: "Strategy is required" }]}
|
||||
>
|
||||
<Select
|
||||
options={availableStrategies.map((s) => ({ label: s, value: s }))}
|
||||
placeholder="Select strategy"
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
{selectedStrategy && strategyDescriptions[selectedStrategy] && (
|
||||
<Paragraph className="text-xs text-gray-500 -mt-2 mb-4">
|
||||
{strategyDescriptions[selectedStrategy]}
|
||||
</Paragraph>
|
||||
)}
|
||||
|
||||
{STRATEGIES_WITH_ARGS.has(String(selectedStrategy)) && (
|
||||
<Form.Item
|
||||
label="Strategy Arguments (JSON)"
|
||||
name="routing_strategy_args"
|
||||
extra={
|
||||
selectedStrategy === "latency-based-routing"
|
||||
? "Example: { \"ttl\": 3600, \"lowest_latency_buffer\": 0 }"
|
||||
: "Example: { \"ttl\": 60 }"
|
||||
}
|
||||
>
|
||||
<Input.TextArea
|
||||
rows={4}
|
||||
placeholder='{ "ttl": 3600 }'
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</Form.Item>
|
||||
)}
|
||||
|
||||
<Space direction="vertical" className="w-full mt-2">
|
||||
<Text type="secondary" className="text-xs">
|
||||
Models not claimed by an explicit group fall through to the proxy's top-level routing
|
||||
strategy.
|
||||
</Text>
|
||||
</Space>
|
||||
</Form>
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default RoutingGroupModal;
|
||||
@@ -0,0 +1,241 @@
|
||||
"use client";
|
||||
|
||||
import React, { useState } from "react";
|
||||
import { Flex, Table, Tabs, Tag, Tooltip, Typography, Button } from "antd";
|
||||
import type { ColumnsType } from "antd/es/table";
|
||||
import { BranchesOutlined, DeleteOutlined, EditOutlined, CodeOutlined } from "@ant-design/icons";
|
||||
import type { RoutingGroup } from "./types";
|
||||
|
||||
const { Text, Paragraph } = Typography;
|
||||
|
||||
interface RoutingGroupsTableProps {
|
||||
groups: RoutingGroup[];
|
||||
loading?: boolean;
|
||||
onEdit: (group: RoutingGroup) => void;
|
||||
onDelete: (group: RoutingGroup) => void;
|
||||
proxyBaseUrl?: string;
|
||||
}
|
||||
|
||||
const formatStrategyLabel = (strategy: string): string => {
|
||||
switch (strategy) {
|
||||
case "simple-shuffle":
|
||||
return "Simple Shuffle";
|
||||
case "least-busy":
|
||||
return "Least Busy";
|
||||
case "usage-based-routing":
|
||||
return "Usage Based";
|
||||
case "latency-based-routing":
|
||||
return "Latency Based";
|
||||
default:
|
||||
return strategy;
|
||||
}
|
||||
};
|
||||
|
||||
const resolveBaseUrl = (proxyBaseUrl?: string): string => {
|
||||
if (proxyBaseUrl && proxyBaseUrl.trim()) return proxyBaseUrl;
|
||||
if (typeof window !== "undefined" && window.location?.origin) return window.location.origin;
|
||||
return "<your_proxy_base_url>";
|
||||
};
|
||||
|
||||
const exampleModel = (group: RoutingGroup): string => group.models[0] ?? "<your-model>";
|
||||
|
||||
const buildCurlSnippet = (group: RoutingGroup, baseUrl: string): string =>
|
||||
`curl -X POST '${baseUrl}/v1/chat/completions' \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-H 'Authorization: Bearer $LITELLM_API_KEY' \\
|
||||
-d '{
|
||||
"model": "${exampleModel(group)}",
|
||||
"messages": [{"role": "user", "content": "Hello!"}]
|
||||
}'`;
|
||||
|
||||
const buildPythonSnippet = (group: RoutingGroup, baseUrl: string): string =>
|
||||
`from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="$LITELLM_API_KEY",
|
||||
base_url="${baseUrl}",
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="${exampleModel(group)}",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
)
|
||||
|
||||
print(response)`;
|
||||
|
||||
const buildJsSnippet = (group: RoutingGroup, baseUrl: string): string =>
|
||||
`import OpenAI from "openai";
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: process.env.LITELLM_API_KEY,
|
||||
baseURL: "${baseUrl}",
|
||||
});
|
||||
|
||||
const response = await client.chat.completions.create({
|
||||
model: "${exampleModel(group)}",
|
||||
messages: [{ role: "user", content: "Hello!" }],
|
||||
});
|
||||
|
||||
console.log(response);`;
|
||||
|
||||
interface RoutingGroupSnippetProps {
|
||||
group: RoutingGroup;
|
||||
baseUrl: string;
|
||||
}
|
||||
|
||||
const SNIPPET_BLOCK_STYLE: React.CSSProperties = {
|
||||
backgroundColor: "#111827",
|
||||
color: "#f3f4f6",
|
||||
borderRadius: 6,
|
||||
padding: 16,
|
||||
fontSize: 12,
|
||||
whiteSpace: "pre",
|
||||
overflowX: "auto",
|
||||
};
|
||||
|
||||
const RoutingGroupSnippet: React.FC<RoutingGroupSnippetProps> = ({ group, baseUrl }) => {
|
||||
const snippets = {
|
||||
curl: buildCurlSnippet(group, baseUrl),
|
||||
python: buildPythonSnippet(group, baseUrl),
|
||||
javascript: buildJsSnippet(group, baseUrl),
|
||||
} as const;
|
||||
type SnippetKey = keyof typeof snippets;
|
||||
const [activeKey, setActiveKey] = useState<SnippetKey>("curl");
|
||||
|
||||
const items = [
|
||||
{ key: "curl", label: "cURL" },
|
||||
{ key: "python", label: "Python (OpenAI SDK)" },
|
||||
{ key: "javascript", label: "JavaScript (OpenAI SDK)" },
|
||||
].map(({ key, label }) => ({
|
||||
key,
|
||||
label,
|
||||
children: (
|
||||
<Paragraph code className="!mb-0" style={SNIPPET_BLOCK_STYLE}>
|
||||
{snippets[key as SnippetKey]}
|
||||
</Paragraph>
|
||||
),
|
||||
}));
|
||||
|
||||
return (
|
||||
<Tabs
|
||||
size="small"
|
||||
activeKey={activeKey}
|
||||
onChange={(k) => setActiveKey(k as SnippetKey)}
|
||||
items={items}
|
||||
tabBarExtraContent={
|
||||
<Paragraph
|
||||
copyable={{ text: snippets[activeKey], tooltips: ["Copy", "Copied"] }}
|
||||
className="!mb-0"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const RoutingGroupsTable: React.FC<RoutingGroupsTableProps> = ({
|
||||
groups,
|
||||
loading,
|
||||
onEdit,
|
||||
onDelete,
|
||||
proxyBaseUrl,
|
||||
}) => {
|
||||
const [expandedRowKeys, setExpandedRowKeys] = useState<React.Key[]>([]);
|
||||
const baseUrl = resolveBaseUrl(proxyBaseUrl);
|
||||
|
||||
const columns: ColumnsType<RoutingGroup> = [
|
||||
{
|
||||
title: "GROUP NAME",
|
||||
dataIndex: "group_name",
|
||||
key: "group_name",
|
||||
render: (name: string) => (
|
||||
<Text strong className="text-blue-600">
|
||||
{name}
|
||||
</Text>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "MODELS",
|
||||
dataIndex: "models",
|
||||
key: "models",
|
||||
render: (models: string[]) => (
|
||||
<Flex wrap="wrap" gap={4}>
|
||||
{models.map((m) => (
|
||||
<Tag key={m}>{m}</Tag>
|
||||
))}
|
||||
</Flex>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "STRATEGY",
|
||||
dataIndex: "routing_strategy",
|
||||
key: "routing_strategy",
|
||||
render: (strategy: string) => (
|
||||
<span className="inline-flex items-center gap-1.5">
|
||||
<BranchesOutlined className="text-gray-400" />
|
||||
<Text>{formatStrategyLabel(strategy)}</Text>
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "ACTIONS",
|
||||
key: "actions",
|
||||
width: 120,
|
||||
align: "right",
|
||||
render: (_, group) => (
|
||||
<Flex justify="flex-end" align="center" gap={8}>
|
||||
<Tooltip title="Edit">
|
||||
<Button
|
||||
type="text"
|
||||
icon={<EditOutlined />}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onEdit(group);
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Tooltip title="Delete">
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
icon={<DeleteOutlined />}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onDelete(group);
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
</Flex>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Table<RoutingGroup>
|
||||
rowKey="group_name"
|
||||
columns={columns}
|
||||
dataSource={groups}
|
||||
loading={loading}
|
||||
pagination={false}
|
||||
expandable={{
|
||||
expandedRowKeys,
|
||||
onExpandedRowsChange: (keys) => setExpandedRowKeys([...keys]),
|
||||
expandedRowRender: (group) => (
|
||||
<div className="bg-gray-50 border border-gray-200 rounded-md p-4 my-2">
|
||||
<Flex align="center" gap={8} className="mb-2">
|
||||
<CodeOutlined className="text-blue-500" />
|
||||
<Text strong>How routing works for this group</Text>
|
||||
</Flex>
|
||||
<Paragraph className="text-sm text-gray-600 mb-3">
|
||||
Callers request any model in the group by name — LiteLLM picks a deployment behind the
|
||||
scenes using the{" "}
|
||||
<Text strong>{formatStrategyLabel(group.routing_strategy)}</Text> strategy.
|
||||
</Paragraph>
|
||||
<RoutingGroupSnippet group={group} baseUrl={baseUrl} />
|
||||
</div>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default RoutingGroupsTable;
|
||||
@@ -0,0 +1,177 @@
|
||||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { Button, Card, Flex, Input, Modal, Space, Typography } from "antd";
|
||||
import { PlusOutlined, ReloadOutlined, SearchOutlined } from "@ant-design/icons";
|
||||
import { useRoutingGroups, useSaveRoutingGroups } from "@/app/(dashboard)/hooks/routingGroups/useRoutingGroups";
|
||||
import { useRouterFields } from "@/app/(dashboard)/hooks/router/useRouterFields";
|
||||
import { useModelHub } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings";
|
||||
import RoutingGroupsTable from "./RoutingGroupsTable";
|
||||
import RoutingGroupModal from "./RoutingGroupModal";
|
||||
import NotificationsManager from "../molecules/notifications_manager";
|
||||
import type { RoutingGroup } from "./types";
|
||||
|
||||
const { Text } = Typography;
|
||||
|
||||
const RoutingGroups: React.FC = () => {
|
||||
const { data, isLoading, refetch, isFetching } = useRoutingGroups();
|
||||
const { data: routerFields } = useRouterFields();
|
||||
const { data: modelHub } = useModelHub();
|
||||
const proxySettings = useProxySettings();
|
||||
const saveMutation = useSaveRoutingGroups();
|
||||
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [drawerOpen, setDrawerOpen] = useState(false);
|
||||
const [drawerMode, setDrawerMode] = useState<"create" | "edit">("create");
|
||||
const [editingGroup, setEditingGroup] = useState<RoutingGroup | null>(null);
|
||||
const [deletingGroup, setDeletingGroup] = useState<RoutingGroup | null>(null);
|
||||
|
||||
const groups = data?.routingGroups ?? [];
|
||||
|
||||
const filteredGroups = useMemo(() => {
|
||||
const q = searchQuery.trim().toLowerCase();
|
||||
if (!q) return groups;
|
||||
return groups.filter(
|
||||
(g) =>
|
||||
g.group_name.toLowerCase().includes(q) ||
|
||||
g.routing_strategy.toLowerCase().includes(q) ||
|
||||
g.models.some((m) => m.toLowerCase().includes(q)),
|
||||
);
|
||||
}, [groups, searchQuery]);
|
||||
|
||||
const availableStrategies = useMemo(() => {
|
||||
if (data?.availableStrategies?.length) return data.availableStrategies;
|
||||
const fromFields = routerFields?.fields?.find((f) => f.field_name === "routing_strategy")?.options;
|
||||
return fromFields ?? [];
|
||||
}, [data?.availableStrategies, routerFields]);
|
||||
|
||||
const strategyDescriptions = routerFields?.routing_strategy_descriptions ?? {};
|
||||
|
||||
const modelOptions = useMemo<string[]>(() => {
|
||||
const records = (modelHub?.data ?? []) as Array<{ model_group?: string }>;
|
||||
const names = records.map((r) => r.model_group).filter((n): n is string => Boolean(n));
|
||||
return Array.from(new Set(names));
|
||||
}, [modelHub]);
|
||||
|
||||
const openCreate = () => {
|
||||
setDrawerMode("create");
|
||||
setEditingGroup(null);
|
||||
setDrawerOpen(true);
|
||||
};
|
||||
|
||||
const openEdit = (group: RoutingGroup) => {
|
||||
setDrawerMode("edit");
|
||||
setEditingGroup(group);
|
||||
setDrawerOpen(true);
|
||||
};
|
||||
|
||||
const handleSubmit = async (incoming: RoutingGroup) => {
|
||||
const next: RoutingGroup[] =
|
||||
drawerMode === "create"
|
||||
? [...groups, incoming]
|
||||
: groups.map((g) => (g.group_name === editingGroup?.group_name ? incoming : g));
|
||||
|
||||
try {
|
||||
await saveMutation.mutateAsync(next);
|
||||
NotificationsManager.success(
|
||||
drawerMode === "create"
|
||||
? `Created routing group "${incoming.group_name}"`
|
||||
: `Updated routing group "${incoming.group_name}"`,
|
||||
);
|
||||
setDrawerOpen(false);
|
||||
} catch (err) {
|
||||
NotificationsManager.error(
|
||||
err instanceof Error ? err.message : "Failed to save routing group",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const confirmDelete = async () => {
|
||||
if (!deletingGroup) return;
|
||||
const next = groups.filter((g) => g.group_name !== deletingGroup.group_name);
|
||||
try {
|
||||
await saveMutation.mutateAsync(next);
|
||||
NotificationsManager.success(`Deleted routing group "${deletingGroup.group_name}"`);
|
||||
setDeletingGroup(null);
|
||||
} catch (err) {
|
||||
NotificationsManager.error(
|
||||
err instanceof Error ? err.message : "Failed to delete routing group",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Space direction="vertical" size={16} className="w-full">
|
||||
<Card bodyStyle={{ padding: 16 }}>
|
||||
<Flex justify="space-between" align="center" gap={12} className="mb-4">
|
||||
<Input
|
||||
allowClear
|
||||
prefix={<SearchOutlined className="text-gray-400" />}
|
||||
placeholder="Search groups..."
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
className="max-w-sm"
|
||||
/>
|
||||
<Flex align="center" gap={12}>
|
||||
<Button
|
||||
icon={<ReloadOutlined />}
|
||||
onClick={() => refetch()}
|
||||
loading={isFetching && !isLoading}
|
||||
>
|
||||
Refresh
|
||||
</Button>
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}>
|
||||
Create Group
|
||||
</Button>
|
||||
<Text type="secondary" className="text-sm whitespace-nowrap">
|
||||
Showing {filteredGroups.length} {filteredGroups.length === 1 ? "result" : "results"}
|
||||
</Text>
|
||||
</Flex>
|
||||
</Flex>
|
||||
|
||||
<RoutingGroupsTable
|
||||
groups={filteredGroups}
|
||||
loading={isLoading}
|
||||
onEdit={openEdit}
|
||||
onDelete={(g) => setDeletingGroup(g)}
|
||||
proxyBaseUrl={
|
||||
proxySettings.LITELLM_UI_API_DOC_BASE_URL?.trim() ||
|
||||
proxySettings.PROXY_BASE_URL ||
|
||||
""
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
|
||||
<RoutingGroupModal
|
||||
open={drawerOpen}
|
||||
mode={drawerMode}
|
||||
initialValue={editingGroup}
|
||||
availableStrategies={availableStrategies}
|
||||
strategyDescriptions={strategyDescriptions}
|
||||
modelOptions={modelOptions}
|
||||
existingGroupNames={groups.map((g) => g.group_name)}
|
||||
onClose={() => setDrawerOpen(false)}
|
||||
onSubmit={handleSubmit}
|
||||
saving={saveMutation.isPending}
|
||||
/>
|
||||
|
||||
<Modal
|
||||
open={Boolean(deletingGroup)}
|
||||
title="Delete routing group?"
|
||||
okText="Delete"
|
||||
okButtonProps={{ danger: true, loading: saveMutation.isPending }}
|
||||
cancelText="Cancel"
|
||||
onOk={confirmDelete}
|
||||
onCancel={() => setDeletingGroup(null)}
|
||||
>
|
||||
<Text>
|
||||
Models in <Text strong>{deletingGroup?.group_name}</Text> will fall back to the proxy's
|
||||
top-level routing strategy. This cannot be undone.
|
||||
</Text>
|
||||
</Modal>
|
||||
</Space>
|
||||
);
|
||||
};
|
||||
|
||||
export default RoutingGroups;
|
||||
@@ -0,0 +1,12 @@
|
||||
export type RoutingStrategy =
|
||||
| "simple-shuffle"
|
||||
| "least-busy"
|
||||
| "usage-based-routing"
|
||||
| "latency-based-routing";
|
||||
|
||||
export interface RoutingGroup {
|
||||
group_name: string;
|
||||
models: string[];
|
||||
routing_strategy: RoutingStrategy | string;
|
||||
routing_strategy_args?: Record<string, unknown> | null;
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"jsx": "react-jsx",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user