Fix - using managed files w/ OTEL + UI - add model group alias on UI (#13171)

* fix(router.py): safe deep copy kwargs

OTEL adds a parent_otel_span which cannot be deepcopied

* fix: use safe deep copy in other places as well

* test: add script to check and ban copy.deepcopy of kwargs

enforce safe_deep_copy usage

* build(ui/): new component for adding model group alias on UI

* fix(proxy_server.py): support updating model_group_alias via /config/update

allows ui component to work

* fix(router.py): update model_group_alias in router settings based on db value

* fix: fix code qa error
This commit is contained in:
Krish Dholakia
2025-07-31 21:22:04 -07:00
committed by GitHub
parent 547c46cd02
commit c7e4435bdc
13 changed files with 993 additions and 81 deletions
+1
View File
@@ -1388,6 +1388,7 @@ jobs:
- run: python ./tests/documentation_tests/test_circular_imports.py
- run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
+51 -6
View File
@@ -18,24 +18,22 @@ else:
def safe_divide_seconds(
seconds: float,
denominator: float,
default: Optional[float] = None
seconds: float, denominator: float, default: Optional[float] = None
) -> Optional[float]:
"""
Safely divide seconds by denominator, handling zero division.
Args:
seconds: Time duration in seconds
denominator: The divisor (e.g., number of tokens)
default: Value to return if division by zero (defaults to None)
Returns:
The result of the division as a float (seconds per unit), or default if denominator is zero
"""
if denominator <= 0:
return default
return float(seconds / denominator)
@@ -203,3 +201,50 @@ def preserve_upstream_non_openai_attributes(
for key, value in original_chunk.model_dump().items():
if key not in expected_keys:
setattr(model_response, key, value)
def safe_deep_copy(data):
"""
Safe Deep Copy
The LiteLLM Request has some object that can-not be pickled / deep copied
Use this function to safely deep copy the LiteLLM Request
"""
import copy
import litellm
if litellm.safe_memory_mode is True:
return data
litellm_parent_otel_span: Optional[Any] = None
# Step 1: Remove the litellm_parent_otel_span
litellm_parent_otel_span = None
if isinstance(data, dict):
# remove litellm_parent_otel_span since this is not picklable
if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
litellm_parent_otel_span = data["metadata"].pop("litellm_parent_otel_span")
data["metadata"]["litellm_parent_otel_span"] = "placeholder"
if (
"litellm_metadata" in data
and "litellm_parent_otel_span" in data["litellm_metadata"]
):
litellm_parent_otel_span = data["litellm_metadata"].pop(
"litellm_parent_otel_span"
)
data["litellm_metadata"]["litellm_parent_otel_span"] = "placeholder"
new_data = copy.deepcopy(data)
# Step 2: re-add the litellm_parent_otel_span after doing a deep copy
if isinstance(data, dict) and litellm_parent_otel_span is not None:
if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
data["metadata"]["litellm_parent_otel_span"] = litellm_parent_otel_span
if (
"litellm_metadata" in data
and "litellm_parent_otel_span" in data["litellm_metadata"]
):
data["litellm_metadata"][
"litellm_parent_otel_span"
] = litellm_parent_otel_span
return new_data
+2 -2
View File
@@ -1,9 +1,9 @@
import uuid
from copy import deepcopy
from typing import Optional
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
from .asyncify import run_async_function
@@ -41,7 +41,7 @@ async def async_completion_with_fallbacks(**kwargs):
most_recent_exception_str: Optional[str] = None
for fallback in fallbacks:
try:
completion_kwargs = deepcopy(base_kwargs)
completion_kwargs = safe_deep_copy(base_kwargs)
# Handle dictionary fallback configurations
if isinstance(fallback, dict):
model = fallback.pop("model", original_model)
+4 -5
View File
@@ -1,9 +1,8 @@
model_list:
- model_name: genai/test/*
- model_name: "gpt-4o-mini-openai"
litellm_params:
model: openai/*
api_base: https://api.openai.com
model: gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
check_provider_endpoint: true
router_settings:
model_group_alias: {"gpt-4o": "gpt-4o-mini-openai"}
+6 -6
View File
@@ -272,9 +272,6 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
from litellm.proxy.management_endpoints.tag_management_endpoints import (
router as tag_management_router,
)
from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
router as user_agent_analytics_router,
)
from litellm.proxy.management_endpoints.team_callback_endpoints import (
router as team_callback_router,
)
@@ -287,6 +284,9 @@ from litellm.proxy.management_endpoints.ui_sso import (
get_disabled_non_admin_personal_key_creation,
)
from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
router as user_agent_analytics_router,
)
from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
from litellm.proxy.openai_files_endpoints.files_endpoints import (
@@ -2220,7 +2220,9 @@ class ProxyConfig:
litellm_settings = config.get("litellm_settings", {})
mcp_aliases = litellm_settings.get("mcp_aliases", None)
global_mcp_server_manager.load_servers_from_config(mcp_servers_config, mcp_aliases)
global_mcp_server_manager.load_servers_from_config(
mcp_servers_config, mcp_aliases
)
## VECTOR STORES
vector_store_registry_config = config.get("vector_store_registry", None)
@@ -3253,7 +3255,6 @@ async def async_data_generator(
"async_data_generator: received streaming chunk - {}".format(chunk)
)
### CALL HOOKS ### - modify outgoing data
chunk = await proxy_logging_obj.async_post_call_streaming_hook(
user_api_key_dict=user_api_key_dict,
@@ -3262,7 +3263,6 @@ async def async_data_generator(
str_so_far=str_so_far,
)
if isinstance(chunk, (ModelResponse, ModelResponseStream)):
response_str = litellm.get_response_string(response_obj=chunk)
str_so_far += response_str
+41 -56
View File
@@ -52,11 +52,6 @@ from litellm import (
ModelResponseStream,
Router,
)
from litellm.types.mcp import (
MCPPreCallRequestObject,
MCPPreCallResponseObject,
MCPDuringCallResponseObject,
)
from litellm._logging import verbose_proxy_logger
from litellm._service_logger import ServiceLogging, ServiceTypes
from litellm.caching.caching import DualCache, RedisCache
@@ -93,6 +88,11 @@ from litellm.proxy.hooks.parallel_request_limiter import (
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
from litellm.types.mcp import (
MCPDuringCallResponseObject,
MCPPreCallRequestObject,
MCPPreCallResponseObject,
)
from litellm.types.utils import CallTypes, LLMResponseTypes, LoggedLiteLLMParams
if TYPE_CHECKING:
@@ -118,33 +118,6 @@ def print_verbose(print_statement):
print(f"LiteLLM Proxy: {print_statement}") # noqa
def safe_deep_copy(data):
"""
Safe Deep Copy
The LiteLLM Request has some object that can-not be pickled / deep copied
Use this function to safely deep copy the LiteLLM Request
"""
if litellm.safe_memory_mode is True:
return data
litellm_parent_otel_span: Optional[Any] = None
# Step 1: Remove the litellm_parent_otel_span
litellm_parent_otel_span = None
if isinstance(data, dict):
# remove litellm_parent_otel_span since this is not picklable
if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
litellm_parent_otel_span = data["metadata"].pop("litellm_parent_otel_span")
new_data = copy.deepcopy(data)
# Step 2: re-add the litellm_parent_otel_span after doing a deep copy
if isinstance(data, dict) and litellm_parent_otel_span is not None:
if "metadata" in data:
data["metadata"]["litellm_parent_otel_span"] = litellm_parent_otel_span
return new_data
class InternalUsageCache:
def __init__(self, dual_cache: DualCache):
self.dual_cache: DualCache = dual_cache
@@ -474,11 +447,11 @@ class ProxyLogging:
)
async def async_pre_mcp_tool_call_hook(
self,
kwargs: dict,
request_obj: Any,
start_time: datetime,
end_time: datetime,
self,
kwargs: dict,
request_obj: Any,
start_time: datetime,
end_time: datetime,
) -> Optional[Any]:
"""
Pre MCP Tool Call Hook
@@ -489,7 +462,7 @@ class ProxyLogging:
from litellm.types.mcp import MCPPreCallRequestObject, MCPPreCallResponseObject
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=getattr(self, 'dynamic_success_callbacks', None),
dynamic_success_callbacks=getattr(self, "dynamic_success_callbacks", None),
global_callbacks=litellm.success_callback,
)
@@ -500,7 +473,7 @@ class ProxyLogging:
arguments=kwargs.get("arguments", {}),
server_name=kwargs.get("server_name"),
user_api_key_auth=kwargs.get("user_api_key_auth"),
hidden_params=HiddenParams()
hidden_params=HiddenParams(),
)
for callback in callbacks:
@@ -537,10 +510,10 @@ class ProxyLogging:
return global_callbacks
return list(set(dynamic_success_callbacks + global_callbacks))
def _parse_pre_mcp_call_hook_response(
self, response: MCPPreCallResponseObject, original_request: MCPPreCallRequestObject
self,
response: MCPPreCallResponseObject,
original_request: MCPPreCallRequestObject,
) -> Dict[str, Any]:
"""
Parse the response from the pre_mcp_tool_call_hook
@@ -551,18 +524,19 @@ class ProxyLogging:
"""
result = {
"should_proceed": response.should_proceed,
"modified_arguments": response.modified_arguments or original_request.arguments,
"modified_arguments": response.modified_arguments
or original_request.arguments,
"error_message": response.error_message,
"hidden_params": response.hidden_params,
}
return result
async def async_during_mcp_tool_call_hook(
self,
kwargs: dict,
request_obj: Any,
start_time: datetime,
end_time: datetime,
self,
kwargs: dict,
request_obj: Any,
start_time: datetime,
end_time: datetime,
) -> Optional[Any]:
"""
During MCP Tool Call Hook
@@ -570,10 +544,13 @@ class ProxyLogging:
Use this for concurrent monitoring and validation during tool execution.
"""
from litellm.types.llms.base import HiddenParams
from litellm.types.mcp import MCPDuringCallResponseObject, MCPDuringCallRequestObject
from litellm.types.mcp import (
MCPDuringCallRequestObject,
MCPDuringCallResponseObject,
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=getattr(self, 'dynamic_success_callbacks', None),
dynamic_success_callbacks=getattr(self, "dynamic_success_callbacks", None),
global_callbacks=litellm.success_callback,
)
@@ -584,7 +561,7 @@ class ProxyLogging:
arguments=kwargs.get("arguments", {}),
server_name=kwargs.get("server_name"),
start_time=start_time.timestamp() if start_time else None,
hidden_params=HiddenParams()
hidden_params=HiddenParams(),
)
for callback in callbacks:
@@ -603,7 +580,9 @@ class ProxyLogging:
# this allows for execution control decisions
######################################################################
if response is not None:
return self._parse_during_mcp_call_hook_response(response=response)
return self._parse_during_mcp_call_hook_response(
response=response
)
except Exception as e:
verbose_proxy_logger.exception(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format(
@@ -613,7 +592,7 @@ class ProxyLogging:
return None
def _parse_during_mcp_call_hook_response(
self, response: MCPDuringCallResponseObject
self, response: MCPDuringCallResponseObject
) -> Dict[str, Any]:
"""
Parse the response from the during_mcp_tool_call_hook
@@ -1382,9 +1361,15 @@ class PrismaClient:
from prisma import Prisma # type: ignore
except Exception as e:
verbose_proxy_logger.error(f"Failed to import Prisma client: {e}")
verbose_proxy_logger.error("This usually means 'prisma generate' hasn't been run yet.")
verbose_proxy_logger.error("Please run 'prisma generate' to generate the Prisma client.")
raise Exception("Unable to find Prisma binaries. Please run 'prisma generate' first.")
verbose_proxy_logger.error(
"This usually means 'prisma generate' hasn't been run yet."
)
verbose_proxy_logger.error(
"Please run 'prisma generate' to generate the Prisma client."
)
raise Exception(
"Unable to find Prisma binaries. Please run 'prisma generate' first."
)
if http_client is not None:
self.db = PrismaWrapper(
original_prisma=Prisma(http=http_client),
+8 -2
View File
@@ -2914,7 +2914,9 @@ class Router:
)
async def create_file_for_deployment(deployment: dict) -> OpenAIFileObject:
kwargs_copy = copy.deepcopy(kwargs)
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
kwargs_copy = safe_deep_copy(kwargs)
self._update_kwargs_with_deployment(
deployment=deployment,
kwargs=kwargs_copy,
@@ -3165,6 +3167,8 @@ class Router:
async def try_retrieve_batch(model_name: DeploymentTypedDict):
try:
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
model = model_name["litellm_params"].get("model")
data = model_name["litellm_params"].copy()
custom_llm_provider = data.get("custom_llm_provider")
@@ -3178,7 +3182,7 @@ class Router:
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model=model
)
new_kwargs = copy.deepcopy(kwargs)
new_kwargs = safe_deep_copy(kwargs)
self._update_kwargs_with_deployment(
deployment=cast(dict, model_name),
kwargs=new_kwargs,
@@ -6008,6 +6012,7 @@ class Router:
"context_window_fallbacks",
"model_group_retry_policy",
"retry_policy",
"model_group_alias",
]
for var in vars_to_include:
@@ -6037,6 +6042,7 @@ class Router:
"fallbacks",
"context_window_fallbacks",
"model_group_retry_policy",
"model_group_alias",
]
_int_settings = [
+6 -3
View File
@@ -89,6 +89,7 @@ class UpdateRouterConfig(BaseModel):
retry_after: Optional[float] = None
fallbacks: Optional[List[dict]] = None
context_window_fallbacks: Optional[List[dict]] = None
model_group_alias: Optional[Dict[str, Union[str, Dict]]] = {}
model_config = ConfigDict(protected_namespaces=())
@@ -209,7 +210,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
model_info: Optional[Dict] = None
mock_response: Optional[Union[str, ModelResponse, Exception, Any]] = None
# auto-router params
auto_router_config_path: Optional[str] = None
auto_router_config: Optional[str] = None
@@ -343,7 +343,7 @@ class LiteLLM_Params(GenericLiteLLMParams):
if max_retries is not None and isinstance(max_retries, str):
max_retries = int(max_retries) # cast to int
args["max_retries"] = max_retries
super().__init__(**{ **args, **params })
super().__init__(**{**args, **params})
def __contains__(self, key):
# Define custom behavior for the 'in' operator
@@ -776,9 +776,11 @@ class MockRouterTestingParams:
),
)
class ModelGroupSettings(BaseModel):
forward_client_headers_to_llm_api: Optional[List[str]] = None
class PreRoutingHookResponse(BaseModel):
"""
Response object from the pre-routing hook.
@@ -787,5 +789,6 @@ class PreRoutingHookResponse(BaseModel):
Add fields that you expect to be modified by the pre-routing hook.
"""
model: str
messages: Optional[List[Dict[str, str]]]
messages: Optional[List[Dict[str, str]]]
+3 -1
View File
@@ -681,7 +681,9 @@ def function_setup( # noqa: PLR0915
if add_breadcrumb:
try:
details_to_log = copy.deepcopy(kwargs)
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
details_to_log = safe_deep_copy(kwargs)
except Exception:
details_to_log = kwargs
@@ -0,0 +1,142 @@
import ast
import os
class CopyDeepcopyKwargsDetector(ast.NodeVisitor):
def __init__(self):
self.violations = []
def visit_Call(self, node):
# Check if this is a copy.deepcopy call
if self._is_copy_deepcopy_call(node):
# Check if any argument contains 'kwargs' in its name
for arg in node.args:
if self._is_kwargs_related(arg):
# Get line number and argument name for reporting
arg_name = self._get_arg_name(arg)
self.violations.append(
{
"line": node.lineno,
"arg_name": arg_name,
"full_call": (
ast.unparse(node)
if hasattr(ast, "unparse")
else str(node)
),
}
)
self.generic_visit(node)
def _is_copy_deepcopy_call(self, node):
"""Check if this is a copy.deepcopy() call"""
if isinstance(node.func, ast.Attribute):
# Case: copy.deepcopy()
if (
isinstance(node.func.value, ast.Name)
and node.func.value.id == "copy"
and node.func.attr == "deepcopy"
):
return True
elif isinstance(node.func, ast.Name):
# Case: deepcopy() (if imported as 'from copy import deepcopy')
if node.func.id == "deepcopy":
return True
return False
def _is_kwargs_related(self, arg):
"""Check if the argument is kwargs-related"""
if isinstance(arg, ast.Name):
# Direct variable names containing 'kwargs'
return "kwargs" in arg.id.lower()
elif isinstance(arg, ast.Subscript):
# Handle cases like kwargs['key']
if isinstance(arg.value, ast.Name):
return "kwargs" in arg.value.id.lower()
elif isinstance(arg, ast.Attribute):
# Handle cases like self.kwargs
return "kwargs" in arg.attr.lower()
return False
def _get_arg_name(self, arg):
"""Get a readable name for the argument"""
if isinstance(arg, ast.Name):
return arg.id
elif isinstance(arg, ast.Subscript) and isinstance(arg.value, ast.Name):
return f"{arg.value.id}[...]"
elif isinstance(arg, ast.Attribute):
return f"...{arg.attr}"
else:
return "unknown_kwargs_variable"
def find_copy_deepcopy_kwargs_in_file(file_path):
"""Find copy.deepcopy usage with kwargs in a single file"""
try:
with open(file_path, "r", encoding="utf-8") as file:
tree = ast.parse(file.read(), filename=file_path)
detector = CopyDeepcopyKwargsDetector()
detector.visit(tree)
return detector.violations
except Exception as e:
print(f"Error parsing {file_path}: {e}")
return []
def find_copy_deepcopy_kwargs_in_directory(directory):
"""Find copy.deepcopy usage with kwargs in all Python files in directory"""
violations = {}
for root, _, files in os.walk(directory):
for file in files:
if file.endswith(".py"):
file_path = os.path.join(root, file)
print(f"Checking file: {file_path}")
file_violations = find_copy_deepcopy_kwargs_in_file(file_path)
if file_violations:
violations[file_path] = file_violations
return violations
if __name__ == "__main__":
# Check for copy.deepcopy(kwargs) usage in the litellm directory
directory_path = "./litellm"
violations = find_copy_deepcopy_kwargs_in_directory(directory_path)
print("\n" + "=" * 80)
print("COPY.DEEPCOPY KWARGS VIOLATIONS FOUND:")
print("=" * 80)
if violations:
total_violations = 0
for file_path, file_violations in violations.items():
print(f"\n📁 File: {file_path}")
for violation in file_violations:
total_violations += 1
print(
f" ❌ Line {violation['line']}: copy.deepcopy({violation['arg_name']})"
)
print(f" Full call: {violation['full_call']}")
print(f"\n{'='*80}")
print(f"🚨 TOTAL VIOLATIONS: {total_violations}")
print("🚨 USE safe_deep_copy() INSTEAD OF copy.deepcopy() FOR KWARGS!")
print("🚨 Available imports:")
print(" - from litellm.proxy.utils import safe_deep_copy")
print(" - from litellm.litellm_core_utils.core_helpers import safe_deep_copy")
print("=" * 80)
# Get first violation for the exception message
first_file = list(violations.keys())[0]
first_violation = violations[first_file][0]
raise Exception(
f"🚨 Found {total_violations} copy.deepcopy(kwargs) violations! "
f"First violation: {first_file}:{first_violation['line']} - "
f"copy.deepcopy({first_violation['arg_name']}). "
f"Use safe_deep_copy() instead to handle non-serializable objects like OTEL spans."
)
else:
print("✅ No copy.deepcopy(kwargs) violations found!")
print("✅ All kwargs copying appears to use safe_deep_copy() correctly.")
@@ -0,0 +1,339 @@
import React, { useState, useEffect, useCallback } from "react";
import {
Card,
Title,
Text,
Table,
TableHead,
TableRow,
TableHeaderCell,
TableCell,
TableBody,
} from "@tremor/react";
import { message, Input } from "antd";
import { EditOutlined, DeleteOutlined, SaveOutlined, CloseOutlined } from "@ant-design/icons";
import { ChevronDownIcon, ChevronRightIcon, PlusCircleIcon } from "@heroicons/react/outline";
interface KeyValueItem {
id?: string;
key: string;
value: string;
}
interface GenericKeyValueManagerProps {
title: string;
description: string;
keyLabel: string;
valueLabel: string;
keyPlaceholder: string;
valuePlaceholder: string;
items: KeyValueItem[];
onItemsChange: (items: KeyValueItem[]) => void;
onSave?: () => Promise<void>;
showSaveButton?: boolean;
isCollapsible?: boolean;
defaultExpanded?: boolean;
configExample?: React.ReactNode;
additionalActions?: (item: KeyValueItem) => React.ReactNode;
}
const GenericKeyValueManager: React.FC<GenericKeyValueManagerProps> = ({
title,
description,
keyLabel,
valueLabel,
keyPlaceholder,
valuePlaceholder,
items,
onItemsChange,
onSave,
showSaveButton = true,
isCollapsible = false,
defaultExpanded = true,
configExample,
additionalActions,
}) => {
const [newKey, setNewKey] = useState<string>("");
const [newValue, setNewValue] = useState<string>("");
const [editingItem, setEditingItem] = useState<KeyValueItem | null>(null);
const [editingKey, setEditingKey] = useState<string>("");
const [editingValue, setEditingValue] = useState<string>("");
const [isExpanded, setIsExpanded] = useState(defaultExpanded);
const generateId = () => Math.random().toString(36).substr(2, 9);
const handleAddItem = useCallback(() => {
if (newKey.trim() && newValue.trim()) {
const newItem: KeyValueItem = {
id: generateId(),
key: newKey.trim(),
value: newValue.trim(),
};
onItemsChange([...items, newItem]);
setNewKey("");
setNewValue("");
} else {
message.error(`Please provide both ${keyLabel.toLowerCase()} and ${valueLabel.toLowerCase()}`);
}
}, [newKey, newValue, items, onItemsChange, keyLabel, valueLabel]);
const handleEditItem = useCallback((item: KeyValueItem) => {
setEditingItem({ ...item });
setEditingKey(item.key);
setEditingValue(item.value);
}, []);
const handleSaveEdit = useCallback(() => {
if (editingKey.trim() && editingValue.trim()) {
const updatedItems = items.map((item) =>
item.id === editingItem?.id ? { ...item, key: editingKey.trim(), value: editingValue.trim() } : item
);
onItemsChange(updatedItems);
setEditingItem(null);
setEditingKey("");
setEditingValue("");
} else {
message.error(`Please provide both ${keyLabel.toLowerCase()} and ${valueLabel.toLowerCase()}`);
}
}, [editingKey, editingValue, items, editingItem, onItemsChange, keyLabel, valueLabel]);
const handleCancelEdit = useCallback(() => {
setEditingItem(null);
setEditingKey("");
setEditingValue("");
}, []);
const handleDeleteItem = useCallback((id: string) => {
const updatedItems = items.filter((item) => item.id !== id);
onItemsChange(updatedItems);
}, [items, onItemsChange]);
const handleSave = useCallback(async () => {
if (onSave) {
try {
await onSave();
} catch (error) {
console.error("Failed to save:", error);
}
}
}, [onSave]);
const ContentSection = useCallback(() => (
<div className="space-y-6">
{/* Add New Item Section */}
<Card>
<Title className="mb-4">Add New {keyLabel}</Title>
<div className="grid grid-cols-3 gap-4">
<div>
<label className="block text-xs text-gray-500 mb-1">{keyLabel}</label>
<Input
value={newKey}
onChange={(e) => setNewKey(e.target.value)}
placeholder={keyPlaceholder}
size="middle"
/>
</div>
<div>
<label className="block text-xs text-gray-500 mb-1">{valueLabel}</label>
<Input
value={newValue}
onChange={(e) => setNewValue(e.target.value)}
placeholder={valuePlaceholder}
size="middle"
/>
</div>
<div className="flex items-end">
<button
onClick={handleAddItem}
disabled={!newKey.trim() || !newValue.trim()}
className={`flex items-center px-4 py-2 rounded-md text-sm ${
!newKey.trim() || !newValue.trim()
? "bg-gray-300 text-gray-500 cursor-not-allowed"
: "bg-green-600 text-white hover:bg-green-700"
}`}
>
<PlusCircleIcon className="w-4 h-4 mr-1" />
Add {keyLabel}
</button>
</div>
</div>
</Card>
{/* Manage Existing Items Section */}
<Card>
<div className="flex justify-between items-center mb-4">
<Title>Manage Existing {keyLabel}s</Title>
{showSaveButton && (
<button
onClick={handleSave}
className="bg-blue-600 text-white px-4 py-2 rounded-md hover:bg-blue-700"
>
Save All Changes
</button>
)}
</div>
<div className="rounded-lg custom-border relative">
<div className="overflow-x-auto">
<Table className="[&_td]:py-0.5 [&_th]:py-1">
<TableHead>
<TableRow>
<TableHeaderCell className="py-1 h-8">{keyLabel}</TableHeaderCell>
<TableHeaderCell className="py-1 h-8">{valueLabel}</TableHeaderCell>
<TableHeaderCell className="py-1 h-8">Actions</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{items.map((item) => (
<TableRow key={item.id} className="h-8">
{editingItem && editingItem.id === item.id ? (
<>
<TableCell className="py-0.5">
<Input
value={editingKey}
onChange={(e) => setEditingKey(e.target.value)}
size="small"
/>
</TableCell>
<TableCell className="py-0.5">
<Input
value={editingValue}
onChange={(e) => setEditingValue(e.target.value)}
size="small"
/>
</TableCell>
<TableCell className="py-0.5 whitespace-nowrap">
<div className="flex space-x-2">
<button
onClick={handleSaveEdit}
className="text-xs bg-blue-50 text-blue-600 px-2 py-1 rounded hover:bg-blue-100"
>
Save
</button>
<button
onClick={handleCancelEdit}
className="text-xs bg-gray-50 text-gray-600 px-2 py-1 rounded hover:bg-gray-100"
>
Cancel
</button>
</div>
</TableCell>
</>
) : (
<>
<TableCell className="py-0.5 text-sm text-gray-900">
{item.key}
</TableCell>
<TableCell className="py-0.5 text-sm text-gray-500">
{item.value}
</TableCell>
<TableCell className="py-0.5 whitespace-nowrap">
<div className="flex space-x-2">
{additionalActions && additionalActions(item)}
<button
onClick={() => handleEditItem(item)}
className="text-xs bg-blue-50 text-blue-600 px-2 py-1 rounded hover:bg-blue-100"
>
Edit
</button>
<button
onClick={() => handleDeleteItem(item.id!)}
className="text-xs bg-red-50 text-red-600 px-2 py-1 rounded hover:bg-red-100"
>
Delete
</button>
</div>
</TableCell>
</>
)}
</TableRow>
))}
{items.length === 0 && (
<TableRow>
<TableCell
colSpan={3}
className="py-0.5 text-sm text-gray-500 text-center"
>
No {keyLabel.toLowerCase()}s added yet. Add a new {keyLabel.toLowerCase()} above.
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</div>
</div>
</Card>
{/* Configuration Example */}
{configExample && (
<Card>
<Title className="mb-4">Configuration Example</Title>
{configExample}
</Card>
)}
</div>
), [
keyLabel,
valueLabel,
keyPlaceholder,
valuePlaceholder,
newKey,
newValue,
items,
editingItem,
editingKey,
editingValue,
showSaveButton,
configExample,
additionalActions,
handleAddItem,
handleSave,
handleEditItem,
handleSaveEdit,
handleCancelEdit,
handleDeleteItem,
]);
if (isCollapsible) {
return (
<Card className="mb-6">
<div
className="flex items-center justify-between cursor-pointer"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex flex-col">
<Title className="mb-0">{title}</Title>
<p className="text-sm text-gray-500">{description}</p>
</div>
<div className="flex items-center">
{isExpanded ? (
<ChevronDownIcon className="w-5 h-5 text-gray-500" />
) : (
<ChevronRightIcon className="w-5 h-5 text-gray-500" />
)}
</div>
</div>
{isExpanded && (
<div className="mt-4">
<ContentSection />
</div>
)}
</Card>
);
}
return (
<div>
<div className="mb-6">
<Title>{title}</Title>
<Text className="text-gray-600 mt-2 block">{description}</Text>
</div>
<div>
<ContentSection />
</div>
</div>
);
};
export default GenericKeyValueManager;
@@ -73,6 +73,7 @@ import { ModelDataTable } from "./model_dashboard/table";
import { columns } from "./model_dashboard/columns";
import HealthCheckComponent from "./model_dashboard/HealthCheckComponent";
import PassThroughSettings from "./pass_through_settings";
import ModelGroupAliasSettings from "./model_group_alias_settings";
import { all_admin_roles } from "@/utils/roles";
import { Table as TableInstance } from "@tanstack/react-table";
@@ -197,6 +198,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [credentialsList, setCredentialsList] = useState<CredentialItem[]>([]);
// Model Group Alias state
const [modelGroupAlias, setModelGroupAlias] = useState<{[key: string]: string}>({});
// Add state for advanced settings visibility
const [showAdvancedSettings, setShowAdvancedSettings] =
useState<boolean>(false);
@@ -479,6 +483,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
}
};
useEffect(() => {
if (!accessToken || !token || !userRole || !userID) {
return;
@@ -646,6 +652,10 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
setModelGroupRetryPolicy(model_group_retry_policy);
setGlobalRetryPolicy(router_settings.retry_policy);
setDefaultRetry(default_retries);
// Set model group alias
const model_group_alias = router_settings.model_group_alias || {};
setModelGroupAlias(model_group_alias);
} catch (error) {
console.error("There was an error fetching the model data", error);
}
@@ -1095,6 +1105,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
{all_admin_roles.includes(userRole) && (
<Tab>Model Retry Settings</Tab>
)}
{all_admin_roles.includes(userRole) && (
<Tab>Model Group Alias</Tab>
)}
</div>
<div className="flex items-center space-x-2">
@@ -1859,6 +1872,13 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
Save
</Button>
</TabPanel>
<TabPanel>
<ModelGroupAliasSettings
accessToken={accessToken}
initialModelGroupAlias={modelGroupAlias}
onAliasUpdate={setModelGroupAlias}
/>
</TabPanel>
</TabPanels>
</TabGroup>
)}
@@ -0,0 +1,370 @@
import React, { useState, useEffect } from "react";
import { message } from "antd";
import { PlusCircleIcon, PencilIcon, TrashIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline";
import { setCallbacksCall } from "./networking";
import {
Card,
Title,
Text,
Table,
TableHead,
TableHeaderCell,
TableBody,
TableRow,
TableCell
} from "@tremor/react";
interface ModelGroupAliasSettingsProps {
accessToken: string;
initialModelGroupAlias?: { [key: string]: string };
onAliasUpdate?: (updatedAlias: { [key: string]: string }) => void;
}
interface AliasItem {
id: string;
aliasName: string;
targetModelGroup: string;
}
const ModelGroupAliasSettings: React.FC<ModelGroupAliasSettingsProps> = ({
accessToken,
initialModelGroupAlias = {},
onAliasUpdate,
}) => {
const [aliases, setAliases] = useState<AliasItem[]>([]);
const [newAlias, setNewAlias] = useState({ aliasName: "", targetModelGroup: "" });
const [editingAlias, setEditingAlias] = useState<AliasItem | null>(null);
const [isExpanded, setIsExpanded] = useState(true);
useEffect(() => {
// Convert object to array for display
const aliasArray = Object.entries(initialModelGroupAlias).map(([aliasName, targetModelGroup], index) => ({
id: `${index}-${aliasName}`,
aliasName,
targetModelGroup,
}));
setAliases(aliasArray);
}, [initialModelGroupAlias]);
const saveAliasesToBackend = async (updatedAliases: AliasItem[]) => {
if (!accessToken) {
console.error("Access token is missing");
return false;
}
try {
// Convert array back to object format
const aliasObject: { [key: string]: string } = {};
updatedAliases.forEach(alias => {
aliasObject[alias.aliasName] = alias.targetModelGroup;
});
const payload = {
router_settings: {
model_group_alias: aliasObject,
},
};
console.log("Saving model group alias:", aliasObject);
await setCallbacksCall(accessToken, payload);
if (onAliasUpdate) {
onAliasUpdate(aliasObject);
}
return true;
} catch (error) {
console.error("Failed to save model group alias settings:", error);
message.error("Failed to save model group alias settings");
return false;
}
};
const handleAddAlias = async () => {
if (!newAlias.aliasName || !newAlias.targetModelGroup) {
message.error("Please provide both alias name and target model group");
return;
}
// Check for duplicate alias names
if (aliases.some(alias => alias.aliasName === newAlias.aliasName)) {
message.error("An alias with this name already exists");
return;
}
const newAliasObj: AliasItem = {
id: `${Date.now()}-${newAlias.aliasName}`,
aliasName: newAlias.aliasName,
targetModelGroup: newAlias.targetModelGroup,
};
const updatedAliases = [...aliases, newAliasObj];
if (await saveAliasesToBackend(updatedAliases)) {
setAliases(updatedAliases);
setNewAlias({ aliasName: "", targetModelGroup: "" });
message.success("Alias added successfully");
}
};
const handleEditAlias = (alias: AliasItem) => {
setEditingAlias({ ...alias });
};
const handleUpdateAlias = async () => {
if (!editingAlias) return;
if (!editingAlias.aliasName || !editingAlias.targetModelGroup) {
message.error("Please provide both alias name and target model group");
return;
}
// Check for duplicate alias names (excluding current alias)
if (aliases.some(alias => alias.id !== editingAlias.id && alias.aliasName === editingAlias.aliasName)) {
message.error("An alias with this name already exists");
return;
}
const updatedAliases = aliases.map(alias =>
alias.id === editingAlias.id ? editingAlias : alias
);
if (await saveAliasesToBackend(updatedAliases)) {
setAliases(updatedAliases);
setEditingAlias(null);
message.success("Alias updated successfully");
}
};
const handleCancelEdit = () => {
setEditingAlias(null);
};
const deleteAlias = async (aliasId: string) => {
const updatedAliases = aliases.filter(alias => alias.id !== aliasId);
if (await saveAliasesToBackend(updatedAliases)) {
setAliases(updatedAliases);
message.success("Alias deleted successfully");
}
};
// Convert current aliases to object for config example
const aliasObject = aliases.reduce((acc, alias) => {
acc[alias.aliasName] = alias.targetModelGroup;
return acc;
}, {} as { [key: string]: string });
return (
<Card className="mb-6">
<div
className="flex items-center justify-between cursor-pointer"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex flex-col">
<Title className="mb-0">Model Group Alias Settings</Title>
<p className="text-sm text-gray-500">Create aliases for your model groups to simplify API calls. For example, you can create an alias 'gpt-4o' that points to 'gpt-4o-mini-openai' model group.</p>
</div>
<div className="flex items-center">
{isExpanded ? (
<ChevronDownIcon className="w-5 h-5 text-gray-500" />
) : (
<ChevronRightIcon className="w-5 h-5 text-gray-500" />
)}
</div>
</div>
{isExpanded && (
<div className="mt-4">
<div className="mb-6">
<Text className="text-sm font-medium text-gray-700 mb-2">Add New Alias</Text>
<div className="grid grid-cols-3 gap-4">
<div>
<label className="block text-xs text-gray-500 mb-1">Alias Name</label>
<input
type="text"
value={newAlias.aliasName}
onChange={(e) =>
setNewAlias({
...newAlias,
aliasName: e.target.value,
})
}
placeholder="e.g., gpt-4o"
className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm"
/>
</div>
<div>
<label className="block text-xs text-gray-500 mb-1">
Target Model Group
</label>
<input
type="text"
value={newAlias.targetModelGroup}
onChange={(e) =>
setNewAlias({
...newAlias,
targetModelGroup: e.target.value,
})
}
placeholder="e.g., gpt-4o-mini-openai"
className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm"
/>
</div>
<div className="flex items-end">
<button
onClick={handleAddAlias}
disabled={!newAlias.aliasName || !newAlias.targetModelGroup}
className={`flex items-center px-4 py-2 rounded-md text-sm ${!newAlias.aliasName || !newAlias.targetModelGroup ? 'bg-gray-300 text-gray-500 cursor-not-allowed' : 'bg-green-600 text-white hover:bg-green-700'}`}
>
<PlusCircleIcon className="w-4 h-4 mr-1" />
Add Alias
</button>
</div>
</div>
</div>
<Text className="text-sm font-medium text-gray-700 mb-2">
Manage Existing Aliases
</Text>
<div className="rounded-lg custom-border relative mb-6">
<div className="overflow-x-auto">
<Table className="[&_td]:py-0.5 [&_th]:py-1">
<TableHead>
<TableRow>
<TableHeaderCell className="py-1 h-8">
Alias Name
</TableHeaderCell>
<TableHeaderCell className="py-1 h-8">
Target Model Group
</TableHeaderCell>
<TableHeaderCell className="py-1 h-8">
Actions
</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{aliases.map((alias) => (
<TableRow key={alias.id} className="h-8">
{editingAlias && editingAlias.id === alias.id ? (
<>
<TableCell className="py-0.5">
<input
type="text"
value={editingAlias.aliasName}
onChange={(e) =>
setEditingAlias({
...editingAlias,
aliasName: e.target.value,
})
}
className="w-full px-2 py-1 border border-gray-300 rounded-md text-sm"
/>
</TableCell>
<TableCell className="py-0.5">
<input
type="text"
value={editingAlias.targetModelGroup}
onChange={(e) =>
setEditingAlias({
...editingAlias,
targetModelGroup: e.target.value,
})
}
className="w-full px-2 py-1 border border-gray-300 rounded-md text-sm"
/>
</TableCell>
<TableCell className="py-0.5 whitespace-nowrap">
<div className="flex space-x-2">
<button
onClick={handleUpdateAlias}
className="text-xs bg-blue-50 text-blue-600 px-2 py-1 rounded hover:bg-blue-100"
>
Save
</button>
<button
onClick={handleCancelEdit}
className="text-xs bg-gray-50 text-gray-600 px-2 py-1 rounded hover:bg-gray-100"
>
Cancel
</button>
</div>
</TableCell>
</>
) : (
<>
<TableCell className="py-0.5 text-sm text-gray-900">
{alias.aliasName}
</TableCell>
<TableCell className="py-0.5 text-sm text-gray-500">
{alias.targetModelGroup}
</TableCell>
<TableCell className="py-0.5 whitespace-nowrap">
<div className="flex space-x-2">
<button
onClick={() => handleEditAlias(alias)}
className="text-xs bg-blue-50 text-blue-600 px-2 py-1 rounded hover:bg-blue-100"
>
<PencilIcon className="w-3 h-3" />
</button>
<button
onClick={() => deleteAlias(alias.id)}
className="text-xs bg-red-50 text-red-600 px-2 py-1 rounded hover:bg-red-100"
>
<TrashIcon className="w-3 h-3" />
</button>
</div>
</TableCell>
</>
)}
</TableRow>
))}
{aliases.length === 0 && (
<TableRow>
<TableCell
colSpan={3}
className="py-0.5 text-sm text-gray-500 text-center"
>
No aliases added yet. Add a new alias above.
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</div>
</div>
{/* Configuration Example */}
<Card>
<Title className="mb-4">Configuration Example</Title>
<Text className="text-gray-600 mb-4">
Here's how your current aliases would look in the config.yaml:
</Text>
<div className="bg-gray-100 rounded-lg p-4 font-mono text-sm">
<div className="text-gray-700">
router_settings:
<br />
&nbsp;&nbsp;model_group_alias:
{Object.keys(aliasObject).length === 0 ? (
<span className="text-gray-500">
<br />
&nbsp;&nbsp;&nbsp;&nbsp;# No aliases configured yet
</span>
) : (
Object.entries(aliasObject).map(([key, value]) => (
<span key={key}>
<br />
&nbsp;&nbsp;&nbsp;&nbsp;"{key}": "{value}"
</span>
))
)}
</div>
</div>
</Card>
</div>
)}
</Card>
);
};
export default ModelGroupAliasSettings;