mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-12 21:04:10 +00:00
Merge pull request #18880 from BerriAI/litellm_fs_router_fields
[Infra] Router Fields Endpoint + React Query for Router Fields
This commit is contained in:
yuneng-jiang
@@ -4,6 +4,7 @@ ROUTER SETTINGS MANAGEMENT
|
||||
Endpoints for accessing router configuration and metadata
|
||||
|
||||
GET /router/settings - Get router configuration including available routing strategies
|
||||
GET /router/fields - Get router settings field definitions without values (for UI rendering)
|
||||
"""
|
||||
|
||||
import inspect
|
||||
@@ -37,6 +38,15 @@ class RouterSettingsResponse(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class RouterFieldsResponse(BaseModel):
|
||||
fields: List[RouterSettingsField] = Field(
|
||||
description="List of all configurable router settings with metadata (without field values)"
|
||||
)
|
||||
routing_strategy_descriptions: Dict[str, str] = Field(
|
||||
description="Descriptions for each routing strategy option"
|
||||
)
|
||||
|
||||
|
||||
def _get_routing_strategies_from_router_class() -> List[str]:
|
||||
"""
|
||||
Dynamically extract routing strategies from the Router class __init__ method.
|
||||
@@ -120,3 +130,53 @@ async def get_router_settings(
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.get(
|
||||
"/router/fields",
|
||||
tags=["Router Settings"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RouterFieldsResponse,
|
||||
)
|
||||
async def get_router_fields(
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Get router settings field definitions without values.
|
||||
|
||||
Returns only the field metadata (type, description, default, options) without
|
||||
populating field_value. This is useful for UI components that need to know
|
||||
what fields to render, but will get the actual values from a different endpoint.
|
||||
|
||||
Returns:
|
||||
- fields: List of all configurable router settings with their metadata (type, description, default, options)
|
||||
The routing_strategy field includes available options extracted from the Router class
|
||||
Note: field_value will be None for all fields
|
||||
- routing_strategy_descriptions: Descriptions for each routing strategy option
|
||||
"""
|
||||
try:
|
||||
# Get available routing strategies dynamically from Router class
|
||||
available_routing_strategies = _get_routing_strategies_from_router_class()
|
||||
|
||||
# Get router settings fields from types file
|
||||
router_fields = [field.model_copy(deep=True) for field in ROUTER_SETTINGS_FIELDS]
|
||||
|
||||
# Populate routing_strategy field with available options
|
||||
for field in router_fields:
|
||||
if field.field_name == "routing_strategy":
|
||||
field.options = available_routing_strategies
|
||||
break
|
||||
|
||||
# Ensure field_value is None for all fields (don't populate values)
|
||||
for field in router_fields:
|
||||
field.field_value = None
|
||||
|
||||
return RouterFieldsResponse(
|
||||
fields=router_fields,
|
||||
routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"Error fetching router fields: {str(e)}"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Tests for router settings management endpoints.
|
||||
|
||||
Tests the GET endpoints for router settings and router fields.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../..")
|
||||
)
|
||||
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
class TestRouterSettingsEndpoints:
|
||||
"""Test suite for router settings endpoints"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_fields_success(self):
|
||||
"""
|
||||
Test GET /router/fields endpoint successfully returns field definitions without values.
|
||||
"""
|
||||
# Make request to router fields endpoint
|
||||
response = client.get(
|
||||
"/router/fields",
|
||||
headers={"Authorization": "Bearer sk-1234"}
|
||||
)
|
||||
|
||||
# Verify response
|
||||
assert response.status_code == 200
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Verify response structure
|
||||
assert "fields" in response_data
|
||||
assert "routing_strategy_descriptions" in response_data
|
||||
|
||||
# Verify fields is a list
|
||||
assert isinstance(response_data["fields"], list)
|
||||
assert len(response_data["fields"]) > 0
|
||||
|
||||
# Verify each field has required properties and field_value is None
|
||||
for field in response_data["fields"]:
|
||||
assert "field_name" in field
|
||||
assert "field_type" in field
|
||||
assert "field_description" in field
|
||||
assert "field_default" in field
|
||||
assert "ui_field_name" in field
|
||||
assert "field_value" in field
|
||||
assert field["field_value"] is None # Ensure field_value is None
|
||||
|
||||
# Verify routing_strategy_descriptions is a dict
|
||||
assert isinstance(response_data["routing_strategy_descriptions"], dict)
|
||||
assert len(response_data["routing_strategy_descriptions"]) > 0
|
||||
|
||||
# Verify routing_strategy field has options populated
|
||||
routing_strategy_field = next(
|
||||
(f for f in response_data["fields"] if f["field_name"] == "routing_strategy"),
|
||||
None
|
||||
)
|
||||
assert routing_strategy_field is not None
|
||||
assert "options" in routing_strategy_field
|
||||
assert isinstance(routing_strategy_field["options"], list)
|
||||
assert len(routing_strategy_field["options"]) > 0
|
||||
@@ -0,0 +1,387 @@
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { renderHook, waitFor } from "@testing-library/react";
|
||||
import React, { ReactNode } from "react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RouterFieldsResponse, useRouterFields } from "./useRouterFields";
|
||||
|
||||
// Mock the networking module
|
||||
vi.mock("@/components/networking", () => ({
|
||||
proxyBaseUrl: null,
|
||||
}));
|
||||
|
||||
// Mock useAuthorized hook
|
||||
const mockUseAuthorized = vi.fn();
|
||||
vi.mock("../useAuthorized", () => ({
|
||||
default: () => mockUseAuthorized(),
|
||||
}));
|
||||
|
||||
// Mock global fetch
|
||||
const mockFetch = vi.fn();
|
||||
global.fetch = mockFetch;
|
||||
|
||||
// Mock console methods to avoid noise in tests
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
// Mock data
|
||||
const mockRouterFieldsResponse: RouterFieldsResponse = {
|
||||
fields: [
|
||||
{
|
||||
field_name: "routing_strategy",
|
||||
field_type: "String",
|
||||
field_description: "Routing strategy to use for load balancing across deployments",
|
||||
field_default: "simple-shuffle",
|
||||
options: ["simple-shuffle", "least-busy", "latency-based-routing"],
|
||||
ui_field_name: "Routing Strategy",
|
||||
link: null,
|
||||
},
|
||||
{
|
||||
field_name: "num_retries",
|
||||
field_type: "Integer",
|
||||
field_description: "Number of retries for failed requests",
|
||||
field_default: 0,
|
||||
options: null,
|
||||
ui_field_name: "Number of Retries",
|
||||
link: null,
|
||||
},
|
||||
],
|
||||
routing_strategy_descriptions: {
|
||||
"simple-shuffle": "Randomly picks a deployment from the list. Simple and fast.",
|
||||
"least-busy": "Routes to the deployment with the lowest number of ongoing requests.",
|
||||
"latency-based-routing": "Routes to the deployment with the lowest latency over a sliding window.",
|
||||
},
|
||||
};
|
||||
|
||||
describe("useRouterFields", () => {
|
||||
let queryClient: QueryClient;
|
||||
|
||||
beforeEach(() => {
|
||||
queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
retry: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Reset all mocks
|
||||
vi.clearAllMocks();
|
||||
|
||||
// Set default mock for useAuthorized (enabled state)
|
||||
mockUseAuthorized.mockReturnValue({
|
||||
accessToken: "test-access-token",
|
||||
userRole: "Admin",
|
||||
userId: "test-user-id",
|
||||
token: "test-token",
|
||||
userEmail: "test@example.com",
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: null,
|
||||
showSSOBanner: false,
|
||||
});
|
||||
});
|
||||
|
||||
const wrapper = ({ children }: { children: ReactNode }) =>
|
||||
React.createElement(QueryClientProvider, { client: queryClient }, children);
|
||||
|
||||
it("should render", () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mockRouterFieldsResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
expect(result.current).toBeDefined();
|
||||
});
|
||||
|
||||
it("should return router fields data when query is successful", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mockRouterFieldsResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Initially loading
|
||||
expect(result.current.isLoading).toBe(true);
|
||||
expect(result.current.data).toBeUndefined();
|
||||
|
||||
// Wait for success
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.data).toEqual(mockRouterFieldsResponse);
|
||||
expect(result.current.error).toBeNull();
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockFetch).toHaveBeenCalledWith("/router/fields", {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: "Bearer test-access-token",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle error when fetch fails", async () => {
|
||||
const errorMessage = "Failed to fetch router fields";
|
||||
const errorResponse = { error: errorMessage };
|
||||
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
json: async () => errorResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Initially loading
|
||||
expect(result.current.isLoading).toBe(true);
|
||||
|
||||
// Wait for error
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
expect(result.current.isError).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.error).toBeDefined();
|
||||
expect(result.current.data).toBeUndefined();
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("should not execute query when accessToken is missing", () => {
|
||||
mockUseAuthorized.mockReturnValue({
|
||||
accessToken: null,
|
||||
userRole: "Admin",
|
||||
userId: "test-user-id",
|
||||
token: null,
|
||||
userEmail: "test@example.com",
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: null,
|
||||
showSSOBanner: false,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Query should not execute
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
expect(result.current.data).toBeUndefined();
|
||||
expect(result.current.isFetched).toBe(false);
|
||||
|
||||
// API should not be called
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should not execute query when userId is missing", () => {
|
||||
mockUseAuthorized.mockReturnValue({
|
||||
accessToken: "test-access-token",
|
||||
userRole: "Admin",
|
||||
userId: null,
|
||||
token: "test-token",
|
||||
userEmail: "test@example.com",
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: null,
|
||||
showSSOBanner: false,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Query should not execute
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
expect(result.current.data).toBeUndefined();
|
||||
expect(result.current.isFetched).toBe(false);
|
||||
|
||||
// API should not be called
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should not execute query when userRole is missing", () => {
|
||||
mockUseAuthorized.mockReturnValue({
|
||||
accessToken: "test-access-token",
|
||||
userRole: null,
|
||||
userId: "test-user-id",
|
||||
token: "test-token",
|
||||
userEmail: "test@example.com",
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: null,
|
||||
showSSOBanner: false,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Query should not execute
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
expect(result.current.data).toBeUndefined();
|
||||
expect(result.current.isFetched).toBe(false);
|
||||
|
||||
// API should not be called
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should handle network error", async () => {
|
||||
const networkError = new Error("Network error");
|
||||
mockFetch.mockRejectedValueOnce(networkError);
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
// Wait for error
|
||||
await waitFor(() => {
|
||||
expect(result.current.isError).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.error).toBeDefined();
|
||||
expect(result.current.data).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should use relative URL when proxyBaseUrl is null", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mockRouterFieldsResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
// When proxyBaseUrl is null, should use relative URL
|
||||
expect(mockFetch).toHaveBeenCalledWith("/router/fields", {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: "Bearer test-access-token",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle error response with different error formats", async () => {
|
||||
const errorFormats = [
|
||||
{ error: { message: "Error message" } },
|
||||
{ message: "Error message" },
|
||||
{ detail: "Error detail" },
|
||||
{ error: "Error string" },
|
||||
{ unknown: "format" },
|
||||
];
|
||||
|
||||
for (const errorFormat of errorFormats) {
|
||||
vi.clearAllMocks();
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
json: async () => errorFormat,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isError).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.error).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("should return empty fields array when API returns empty fields", async () => {
|
||||
const emptyResponse: RouterFieldsResponse = {
|
||||
fields: [],
|
||||
routing_strategy_descriptions: {},
|
||||
};
|
||||
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => emptyResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.data?.fields).toEqual([]);
|
||||
expect(result.current.data?.routing_strategy_descriptions).toEqual({});
|
||||
});
|
||||
|
||||
it("should have correct query configuration", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mockRouterFieldsResponse,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
// Verify the query was called
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
|
||||
// The hook should have the expected properties from useQuery
|
||||
expect(result.current).toHaveProperty("data");
|
||||
expect(result.current).toHaveProperty("isLoading");
|
||||
expect(result.current).toHaveProperty("isError");
|
||||
expect(result.current).toHaveProperty("isSuccess");
|
||||
expect(result.current).toHaveProperty("error");
|
||||
});
|
||||
|
||||
it("should handle fields with null options", async () => {
|
||||
const responseWithNullOptions: RouterFieldsResponse = {
|
||||
fields: [
|
||||
{
|
||||
field_name: "timeout",
|
||||
field_type: "Float",
|
||||
field_description: "Timeout for requests in seconds",
|
||||
field_default: null,
|
||||
options: null,
|
||||
ui_field_name: "Timeout",
|
||||
link: null,
|
||||
},
|
||||
],
|
||||
routing_strategy_descriptions: {},
|
||||
};
|
||||
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => responseWithNullOptions,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.data?.fields[0].options).toBeNull();
|
||||
});
|
||||
|
||||
it("should handle fields with link property", async () => {
|
||||
const responseWithLink: RouterFieldsResponse = {
|
||||
fields: [
|
||||
{
|
||||
field_name: "enable_tag_filtering",
|
||||
field_type: "Boolean",
|
||||
field_description: "Enable tag-based routing",
|
||||
field_default: false,
|
||||
options: null,
|
||||
ui_field_name: "Enable Tag Filtering",
|
||||
link: "https://docs.litellm.ai/docs/proxy/tag_routing",
|
||||
},
|
||||
],
|
||||
routing_strategy_descriptions: {},
|
||||
};
|
||||
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => responseWithLink,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useRouterFields(), { wrapper });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isSuccess).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.data?.fields[0].link).toBe("https://docs.litellm.ai/docs/proxy/tag_routing");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { useQuery, UseQueryResult } from "@tanstack/react-query";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
import { proxyBaseUrl } from "@/components/networking";
|
||||
|
||||
export interface RouterSettingsField {
|
||||
field_name: string;
|
||||
field_type: string;
|
||||
field_description: string;
|
||||
field_default: any;
|
||||
options: string[] | null;
|
||||
ui_field_name: string;
|
||||
link: string | null;
|
||||
}
|
||||
|
||||
export interface RouterFieldsResponse {
|
||||
fields: RouterSettingsField[];
|
||||
routing_strategy_descriptions: Record<string, string>;
|
||||
}
|
||||
|
||||
const routerFieldsKeys = createQueryKeys("routerFields");
|
||||
|
||||
const deriveErrorMessage = (errorData: any): string => {
|
||||
return (
|
||||
(errorData?.error && (errorData.error.message || errorData.error)) ||
|
||||
errorData?.message ||
|
||||
errorData?.detail ||
|
||||
errorData?.error ||
|
||||
JSON.stringify(errorData)
|
||||
);
|
||||
};
|
||||
|
||||
const getRouterFields = async (accessToken: string): Promise<RouterFieldsResponse> => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/router/fields` : `/router/fields`;
|
||||
|
||||
console.log("Fetching router fields from:", url);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
const errorMessage = deriveErrorMessage(errorData);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data: RouterFieldsResponse = await response.json();
|
||||
console.log("Fetched router fields:", data);
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch router fields:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const useRouterFields = (): UseQueryResult<RouterFieldsResponse> => {
|
||||
const { accessToken, userId, userRole } = useAuthorized();
|
||||
return useQuery<RouterFieldsResponse>({
|
||||
queryKey: routerFieldsKeys.detail("fields"),
|
||||
queryFn: async () => await getRouterFields(accessToken!),
|
||||
enabled: Boolean(accessToken && userId && userRole),
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user