diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 9a55b0455a..411ca10aa9 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1146,6 +1146,7 @@ async def clear_cache(): try: llm_router.model_list.clear() + llm_router.auto_routers.clear() await proxy_config.add_deployment( prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 02364eb22a..e6f3f7f67e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -7,17 +7,6 @@ model_list: model: openai/* -guardrails: - - guardrail_name: "bedrock-post-guard" - litellm_params: - guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" - mode: "post_call" - guardrailIdentifier: ff6ujrregl1q - guardrailVersion: "DRAFT" - default_on: true - - - litellm_settings: callbacks: ["datadog_llm_observability"] cache: true diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 479406a922..ff4a64085f 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1292,7 +1292,8 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list): # Verify AutoRouter was called with correct parameters mock_auto_router.assert_called_once_with( model_name="test-auto-router", - router_config_path="/path/to/config", + auto_router_config_path="/path/to/config", + auto_router_config=None, default_model="gpt-3.5-turbo", embedding_model="text-embedding-ada-002", litellm_router_instance=router, diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index a59546c023..863fb3b62b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -211,21 +211,6 @@ const AddAutoRouterTab: React.FC = ({ showSearch={true} /> - {showCustomDefaultModel && ( - - { - form.setFieldValue('auto_router_default_model', e.target.value); - }} - /> - - )} {/* Auto Router Embedding Model */} = ({ allowClear /> - {showCustomEmbeddingModel && ( - - { - form.setFieldValue('auto_router_embedding_model', e.target.value); - }} - /> - - )} -
Additional Settings diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 0542049e16..525ce14204 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -101,7 +101,6 @@ const AddModelTab: React.FC = ({ return ( <> - Add new model Add Model @@ -109,6 +108,7 @@ const AddModelTab: React.FC = ({ + Add Model
void; } @@ -26,17 +39,30 @@ const RouterConfigBuilder: React.FC = ({ value, onChange, }) => { - const [routes, setRoutes] = useState(value?.routes || []); + const [routes, setRoutes] = useState([]); const [showJsonPreview, setShowJsonPreview] = useState(false); const [expandedRoutes, setExpandedRoutes] = useState([]); - // Initialize expanded routes for existing routes on mount + // Initialize routes from value prop useEffect(() => { - if (value?.routes && value.routes.length > 0 && expandedRoutes.length === 0) { - const existingRouteIds = value.routes.map((route: any) => route.id || `route-${Math.random()}`); - setExpandedRoutes(existingRouteIds); + if (value?.routes) { + const initializedRoutes = value.routes.map((route: SavedRoute, index: number) => ({ + id: route.id || `route-${index}-${Date.now()}`, + model: route.name || route.model || "", // handle both 'name' and 'model' fields + utterances: route.utterances || [], + description: route.description || "", + score_threshold: route.score_threshold || 0.5, + })); + setRoutes(initializedRoutes); + + // Set expanded routes for existing routes + const routeIds = initializedRoutes.map(route => route.id); + setExpandedRoutes(routeIds); + } else { + setRoutes([]); + setExpandedRoutes([]); } - }, [value?.routes, expandedRoutes.length]); + }, [value]); // Handle adding a new route const addRoute = () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx new file mode 100644 index 0000000000..580f6761fe --- /dev/null +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -0,0 +1,262 @@ +import React, { useEffect, useState } from "react"; +import { Modal, Form, Button, Select as AntdSelect, message } from "antd"; +import { Text, TextInput } from "@tremor/react"; +import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; +import { fetchAvailableModels, ModelGroup } from "../chat_ui/llm_calls/fetch_models"; +import RouterConfigBuilder from "../add_model/router_config_builder"; + +interface EditAutoRouterModalProps { + isVisible: boolean; + onCancel: () => void; + onSuccess: (updatedModel: any) => void; + modelData: any; + accessToken: string; + userRole: string; +} + +const EditAutoRouterModal: React.FC = ({ + isVisible, + onCancel, + onSuccess, + modelData, + accessToken, + userRole, +}) => { + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + const [modelAccessGroups, setModelAccessGroups] = useState([]); + const [modelInfo, setModelInfo] = useState([]); + const [showCustomDefaultModel, setShowCustomDefaultModel] = useState(false); + const [showCustomEmbeddingModel, setShowCustomEmbeddingModel] = useState(false); + const [routerConfig, setRouterConfig] = useState(null); + + useEffect(() => { + if (isVisible && modelData) { + initializeForm(); + } + }, [isVisible, modelData]); + + useEffect(() => { + const fetchModelAccessGroups = async () => { + if (!accessToken) return; + try { + const response = await modelAvailableCall(accessToken, "", "", false, null, true, true); + setModelAccessGroups(response["data"].map((model: any) => model["id"])); + } catch (error) { + console.error("Error fetching model access groups:", error); + } + }; + + const loadModels = async () => { + if (!accessToken) return; + try { + const uniqueModels = await fetchAvailableModels(accessToken); + setModelInfo(uniqueModels); + } catch (error) { + console.error("Error fetching model info:", error); + } + }; + + if (isVisible) { + fetchModelAccessGroups(); + loadModels(); + } + }, [isVisible, accessToken]); + + const initializeForm = () => { + try { + // Parse the auto_router_config if it exists and is a string + let parsedConfig = null; + if (modelData.litellm_params?.auto_router_config) { + if (typeof modelData.litellm_params.auto_router_config === 'string') { + parsedConfig = JSON.parse(modelData.litellm_params.auto_router_config); + } else { + parsedConfig = modelData.litellm_params.auto_router_config; + } + } + + setRouterConfig(parsedConfig); + + // Set form values + form.setFieldsValue({ + auto_router_name: modelData.model_name, + auto_router_default_model: modelData.litellm_params?.auto_router_default_model || '', + auto_router_embedding_model: modelData.litellm_params?.auto_router_embedding_model || '', + model_access_group: modelData.model_info?.access_groups || [], + }); + + // Check if using custom models + const allModelGroups = new Set(modelInfo.map(model => model.model_group)); + setShowCustomDefaultModel(!allModelGroups.has(modelData.litellm_params?.auto_router_default_model)); + setShowCustomEmbeddingModel(!allModelGroups.has(modelData.litellm_params?.auto_router_embedding_model)); + + } catch (error) { + console.error("Error parsing auto router config:", error); + message.error("Error loading auto router configuration"); + } + }; + + const handleSubmit = async () => { + try { + setLoading(true); + const values = await form.validateFields(); + + // Prepare the updated litellm_params + const updatedLitellmParams = { + ...modelData.litellm_params, + auto_router_config: JSON.stringify(routerConfig), + auto_router_default_model: values.auto_router_default_model, + auto_router_embedding_model: values.auto_router_embedding_model || undefined, + }; + + // Prepare updated model_info + const updatedModelInfo = { + ...modelData.model_info, + access_groups: values.model_access_group || [], + }; + + const updateData = { + model_name: values.auto_router_name, + litellm_params: updatedLitellmParams, + model_info: updatedModelInfo, + }; + + await modelPatchUpdateCall(accessToken, updateData, modelData.model_info.id); + + const updatedModelData = { + ...modelData, + model_name: values.auto_router_name, + litellm_params: updatedLitellmParams, + model_info: updatedModelInfo, + }; + + message.success("Auto router configuration updated successfully"); + onSuccess(updatedModelData); + onCancel(); + } catch (error) { + console.error("Error updating auto router:", error); + message.error("Failed to update auto router configuration"); + } finally { + setLoading(false); + } + }; + + const modelOptions = modelInfo.map(model => ({ + value: model.model_group, + label: model.model_group, + })); + + return ( + + Cancel + , + , + ]} + width={1000} + destroyOnClose + > +
+ + Edit the auto router configuration including routing logic, default models, and access settings. + + + + {/* Auto Router Name */} + + + + + {/* Router Configuration Builder */} +
+ { + setRouterConfig(config); + }} + /> +
+ + {/* Default Model */} + + { + setShowCustomDefaultModel(value === 'custom'); + }} + options={[ + ...modelOptions, + { value: 'custom', label: 'Enter custom model name' } + ]} + showSearch={true} + /> + + + + {/* Embedding Model */} + + { + setShowCustomEmbeddingModel(value === 'custom'); + }} + options={[ + ...modelOptions, + { value: 'custom', label: 'Enter custom model name' } + ]} + showSearch={true} + allowClear + /> + + + {/* Model Access Groups - Admin only */} + {userRole === "Admin" && ( + + ({ + value: group, + label: group + }))} + maxTagCount="responsive" + allowClear + /> + + )} + +
+
+ ); +}; + +export default EditAutoRouterModal; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index da15c3a3d4..62958ccb90 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -35,6 +35,7 @@ import ReuseCredentialsModal from "./model_add/reuse_credentials"; import CacheControlSettings from "./add_model/cache_control_settings"; import { CheckIcon, CopyIcon } from "lucide-react"; import { copyToClipboard as utilCopyToClipboard } from "../utils/dataUtils"; +import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal"; interface ModelInfoViewProps { modelId: string; @@ -74,18 +75,20 @@ export default function ModelInfoView({ useState(null); const [showCacheControl, setShowCacheControl] = useState(false); const [copiedStates, setCopiedStates] = useState>({}); + const [isAutoRouterModalOpen, setIsAutoRouterModalOpen] = useState(false); const canEditModel = - userRole === "Admin" || modelData.model_info.created_by === userID; + userRole === "Admin" || modelData?.model_info?.created_by === userID; const isAdmin = userRole === "Admin"; + const isAutoRouter = modelData?.litellm_params?.auto_router_config != null; const usingExistingCredential = - modelData.litellm_params?.litellm_credential_name != null && - modelData.litellm_params?.litellm_credential_name != undefined; + modelData?.litellm_params?.litellm_credential_name != null && + modelData?.litellm_params?.litellm_credential_name != undefined; console.log("usingExistingCredential, ", usingExistingCredential); console.log( "modelData.litellm_params.litellm_credential_name, ", - modelData.litellm_params.litellm_credential_name + modelData?.litellm_params?.litellm_credential_name ); useEffect(() => { @@ -271,6 +274,13 @@ export default function ModelInfoView({ } }; + const handleAutoRouterUpdate = (updatedModel: any) => { + setLocalModelData(updatedModel); + if (onModelUpdate) { + onModelUpdate(updatedModel); + } + }; + return (
@@ -430,15 +440,26 @@ export default function ModelInfoView({
Model Settings - {canEditModel && !isEditing && ( - setIsEditing(true)} - className="flex items-center" - > - Edit Model - - )} +
+ {isAutoRouter && canEditModel && !isEditing && ( + setIsAutoRouterModalOpen(true)} + className="flex items-center" + > + Edit Auto Router + + )} + {canEditModel && !isEditing && ( + setIsEditing(true)} + className="flex items-center" + > + Edit Model + + )} +
{localModelData ? (
{modelData.litellm_params.litellm_credential_name} )} + + {/* Edit Auto Router Modal */} + setIsAutoRouterModalOpen(false)} + onSuccess={handleAutoRouterUpdate} + modelData={localModelData || modelData} + accessToken={accessToken || ""} + userRole={userRole || ""} + />
); }