diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx index 4283693d1b..01dd97505c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx @@ -3,7 +3,7 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useTeams from "@/app/(dashboard)/hooks/useTeams"; import { useState } from "react"; -import ModelsAndEndpointsView from "@/app/(dashboard)/models-and-endpoints/model_dashboard"; +import ModelsAndEndpointsView from "@/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView"; const ModelsAndEndpointsPage = () => { const { token, accessToken, userRole, userId, premiumUser } = useAuthorized(); diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 3b5e2b9761..19f0dabe45 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -10,7 +10,6 @@ import { ThemeProvider } from "@/contexts/ThemeContext"; import UserDashboard from "@/components/user_dashboard"; import OldModelDashboard from "@/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView"; import ViewUserDashboard from "@/components/view_users"; -import TeamsView from "@/app/(dashboard)/teams/TeamsView"; import Organizations from "@/components/organizations"; import { fetchOrganizations } from "@/components/organizations"; import AdminPanel from "@/components/admins"; diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index a9c5681cab..707e493f72 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -1,27 +1,19 @@ import React, { useState, useEffect } from "react"; -import Link from "next/link"; import { Typography } from "antd"; import { teamDeleteCall, - teamUpdateCall, - teamInfoCall, Organization, - DEFAULT_ORGANIZATION, fetchMCPAccessGroups, } from "./networking"; -import TeamMemberModal from "@/components/team/edit_membership"; import { fetchTeams } from "./common_components/fetch_teams"; import { - InformationCircleIcon, PencilAltIcon, - PencilIcon, RefreshIcon, - StatusOnlineIcon, TrashIcon, ChevronDownIcon, ChevronRightIcon, } from "@heroicons/react/outline"; -import { Button as Button2, Modal, Form, Input, Select as Select2, message, Tooltip } from "antd"; +import { Button as Button2, Modal, Form, Input, Select as Select2, Tooltip } from "antd"; import NumericalInput from "./shared/numerical_input"; import { fetchAvailableModelsForTeamOrKey, @@ -31,7 +23,7 @@ import { import { Select, SelectItem } from "@tremor/react"; import { InfoCircleOutlined } from "@ant-design/icons"; import { getGuardrailsList } from "./networking"; -import TeamInfoView, { TeamData } from "@/components/team/team_info"; +import TeamInfoView from "@/components/team/team_info"; import TeamSSOSettings from "@/components/TeamSSOSettings"; import { isAdminRole } from "@/utils/roles"; import { @@ -58,7 +50,6 @@ import { TabPanels, Tab, } from "@tremor/react"; -import { CogIcon } from "@heroicons/react/outline"; import AvailableTeamsPanel from "@/components/team/available_teams"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import PremiumLoggingSettings from "./common_components/PremiumLoggingSettings"; @@ -98,14 +89,10 @@ interface EditTeamModalProps { import { teamCreateCall, - teamMemberAddCall, - teamMemberUpdateCall, Member, - modelAvailableCall, v2TeamListCall, } from "./networking"; import { updateExistingKeys } from "@/utils/dataUtils"; -import { deprecate } from "node:util"; interface TeamInfo { members_with_roles: Member[]; diff --git a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx new file mode 100644 index 0000000000..d13786113e --- /dev/null +++ b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx @@ -0,0 +1,1757 @@ +import React, { useState, useEffect, useRef, useMemo } from "react"; +import { + Card, + Title, + Subtitle, + Table, + TableHead, + TableRow, + TableHeaderCell, + TableCell, + TableBody, + Text, + Grid, + Col, +} from "@tremor/react"; +import { CredentialItem, credentialListCall, CredentialsResponse } from "../networking"; + +import { handleAddModelSubmit } from "../add_model/handle_add_model_submit"; + +import CredentialsPanel from "@/components/model_add/credentials"; +import { getDisplayModelName } from "../view_model/model_name_display"; +import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; +import { Select, SelectItem, DateRangePickerValue } from "@tremor/react"; +import UsageDatePicker from "../shared/usage_date_picker"; +import { + modelInfoCall, + modelCostMap, + healthCheckCall, + modelMetricsCall, + streamingModelMetricsCall, + modelExceptionsCall, + modelMetricsSlowResponsesCall, + getCallbacksCall, + setCallbacksCall, + modelSettingsCall, + adminGlobalActivityExceptions, + adminGlobalActivityExceptionsPerDeployment, + allEndUsersCall, +} from "../networking"; +import { BarChart, AreaChart } from "@tremor/react"; +import { Popover, Form, InputNumber } from "antd"; +import { Button } from "@tremor/react"; +import { Typography } from "antd"; +import { RefreshIcon, FilterIcon } from "@heroicons/react/outline"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import type { UploadProps } from "antd"; +import TimeToFirstToken from "../model_metrics/time_to_first_token"; +import { Team } from "../key_team_helpers/key_list"; +import TeamInfoView from "../team/team_info"; +import { Providers, provider_map, getPlaceholder, getProviderModels } from "../provider_info_helpers"; +import ModelInfoView from "../model_info_view"; +import AddModelTab from "../add_model/add_model_tab"; + +import { ModelDataTable } from "../model_dashboard/table"; +import { columns } from "../molecules/models/columns"; +import PriceDataReload from "../price_data_reload"; +import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; +import PassThroughSettings from "../pass_through_settings"; +import ModelGroupAliasSettings from "../model_group_alias_settings"; +import { all_admin_roles } from "@/utils/roles"; +import { Table as TableInstance, PaginationState } from "@tanstack/react-table"; +import NotificationsManager from "../molecules/notifications_manager"; + +interface ModelDashboardProps { + accessToken: string | null; + token: string | null; + userRole: string | null; + userID: string | null; + modelData: any; + keys: any[] | null; + setModelData: any; + premiumUser: boolean; + teams: Team[] | null; +} + +interface RetryPolicyObject { + [key: string]: { [retryPolicyKey: string]: number } | undefined; +} + +interface GlobalRetryPolicyObject { + [retryPolicyKey: string]: number; +} + +interface GlobalExceptionActivityData { + sum_num_rate_limit_exceptions: number; + daily_data: { date: string; num_rate_limit_exceptions: number }[]; +} + +//["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"] + +interface ProviderFields { + field_name: string; + field_type: string; + field_description: string; + field_value: string; +} + +interface ProviderSettings { + name: string; + fields: ProviderFields[]; +} + +const retry_policy_map: Record = { + "BadRequestError (400)": "BadRequestErrorRetries", + "AuthenticationError (401)": "AuthenticationErrorRetries", + "TimeoutError (408)": "TimeoutErrorRetries", + "RateLimitError (429)": "RateLimitErrorRetries", + "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", + "InternalServerError (500)": "InternalServerErrorRetries", +}; + +const OldModelDashboard: React.FC = ({ + accessToken, + token, + userRole, + userID, + modelData = { data: [] }, + keys, + setModelData, + premiumUser, + teams, +}) => { + const [addModelForm] = Form.useForm(); + const [autoRouterForm] = Form.useForm(); + const [modelMap, setModelMap] = useState(null); + const [lastRefreshed, setLastRefreshed] = useState(""); + + const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array + + const [providerSettings, setProviderSettings] = useState([]); + const [selectedProvider, setSelectedProvider] = useState(Providers.OpenAI); + const [healthCheckResponse, setHealthCheckResponse] = useState(null); + const [isHealthCheckLoading, setIsHealthCheckLoading] = useState(false); + const [editModalVisible, setEditModalVisible] = useState(false); + + const [selectedModel, setSelectedModel] = useState(null); + const [availableModelGroups, setAvailableModelGroups] = useState>([]); + const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState>([]); + const [selectedModelGroup, setSelectedModelGroup] = useState(null); + const [modelMetrics, setModelMetrics] = useState([]); + const [modelMetricsCategories, setModelMetricsCategories] = useState([]); + const [streamingModelMetrics, setStreamingModelMetrics] = useState([]); + const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = useState([]); + const [modelExceptions, setModelExceptions] = useState([]); + const [allExceptions, setAllExceptions] = useState([]); + const [slowResponsesData, setSlowResponsesData] = useState([]); + const [dateValue, setDateValue] = useState({ + from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + to: new Date(), + }); + + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); + const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null); + const [defaultRetry, setDefaultRetry] = useState(0); + + const [globalExceptionData, setGlobalExceptionData] = useState( + {} as GlobalExceptionActivityData, + ); + const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState([]); + + const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); + const [selectedAPIKey, setSelectedAPIKey] = useState(null); + const [selectedCustomer, setSelectedCustomer] = useState(null); + + const [allEndUsers, setAllEndUsers] = useState([]); + + const [credentialsList, setCredentialsList] = useState([]); + + // Model Group Alias state + const [modelGroupAlias, setModelGroupAlias] = useState<{ [key: string]: string }>({}); + + // Add state for advanced settings visibility + const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); + + // Add these state variables + const [selectedModelId, setSelectedModelId] = useState(null); + const [editModel, setEditModel] = useState(false); + + const [selectedTeamId, setSelectedTeamId] = useState(null); + const [selectedTeam, setSelectedTeam] = useState(null); + + const [selectedTeamFilter, setSelectedTeamFilter] = useState(null); + const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState(null); + + const [modelNameSearch, setModelNameSearch] = useState(""); + + // Add new state for current team and model view mode + const [currentTeam, setCurrentTeam] = useState("personal"); // 'personal' or team_id + const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">("current_team"); + + // Add state for showing/hiding filters + const [showFilters, setShowFilters] = useState(false); + + const [showColumnDropdown, setShowColumnDropdown] = useState(false); + + const [isDropdownOpen, setIsDropdownOpen] = useState(false); + const [expandedRows, setExpandedRows] = useState>(new Set()); + const dropdownRef = useRef(null); + const tableRef = useRef>(null); + + // Pagination state + const [pagination, setPagination] = useState({ + pageIndex: 0, + pageSize: 50, + }); + const [selectedTabIndex, setSelectedTabIndex] = useState(0); + + const handleCreateNewModelClick = () => { + if (selectedModelId) { + setSelectedModelId(null); + } + setSelectedTabIndex(1); + }; + + const resetFilters = () => { + setModelNameSearch(""); + setSelectedModelGroup("all"); + setSelectedModelAccessGroupFilter(null); + setCurrentTeam("personal"); + setModelViewMode("current_team"); + setPagination({ pageIndex: 0, pageSize: 50 }); + }; + + // Memoize filtered data to prevent unnecessary re-calculations + const filteredData = useMemo(() => { + if (!modelData || !modelData.data || modelData.data.length === 0) { + return []; + } + + return modelData.data.filter((model: any) => { + const searchMatch = + modelNameSearch === "" || model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + + const modelNameMatch = + selectedModelGroup === "all" || + model.model_name === selectedModelGroup || + !selectedModelGroup || + (selectedModelGroup === "wildcard" && model.model_name?.includes("*")); + + const accessGroupMatch = + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter; + + let teamAccessMatch = true; + if (modelViewMode === "current_team") { + if (currentTeam === "personal") { + teamAccessMatch = model.model_info?.direct_access === true; + } else { + teamAccessMatch = model.model_info?.access_via_team_ids?.includes(currentTeam) === true; + } + } + + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch; + }); + }, [modelData, modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); + + // Memoize paginated data + const paginatedData = useMemo(() => { + const startIndex = pagination.pageIndex * pagination.pageSize; + const endIndex = startIndex + pagination.pageSize; + return filteredData.slice(startIndex, endIndex); + }, [filteredData, pagination.pageIndex, pagination.pageSize]); + + // Reset pagination when filters change + useEffect(() => { + setPagination((prev) => ({ ...prev, pageIndex: 0 })); + }, [modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); + + const setProviderModelsFn = (provider: Providers) => { + const _providerModels = getProviderModels(provider, modelMap); + setProviderModels(_providerModels); + console.log(`providerModels: ${_providerModels}`); + }; + + const updateModelMetrics = async ( + modelGroup: string | null, + startTime: Date | undefined, + endTime: Date | undefined, + ) => { + console.log("Updating model metrics for group:", modelGroup); + if (!accessToken || !userID || !userRole || !startTime || !endTime) { + return; + } + console.log("inside updateModelMetrics - startTime:", startTime, "endTime:", endTime); + setSelectedModelGroup(modelGroup); + + let selected_token = selectedAPIKey?.token; + if (selected_token === undefined) { + selected_token = null; + } + + let selected_customer = selectedCustomer; + if (selected_customer === undefined) { + selected_customer = null; + } + + try { + const modelMetricsResponse = await modelMetricsCall( + accessToken, + userID, + userRole, + modelGroup, + startTime.toISOString(), + endTime.toISOString(), + selected_token, + selected_customer, + ); + console.log("Model metrics response:", modelMetricsResponse); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + + const streamingModelMetricsResponse = await streamingModelMetricsCall( + accessToken, + modelGroup, + startTime.toISOString(), + endTime.toISOString(), + ); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + setStreamingModelMetrics(streamingModelMetricsResponse.data); + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases); + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + modelGroup, + startTime.toISOString(), + endTime.toISOString(), + selected_token, + selected_customer, + ); + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + + const slowResponses = await modelMetricsSlowResponsesCall( + accessToken, + userID, + userRole, + modelGroup, + startTime.toISOString(), + endTime.toISOString(), + selected_token, + selected_customer, + ); + + console.log("slowResponses:", slowResponses); + + setSlowResponsesData(slowResponses); + + if (modelGroup) { + const dailyExceptions = await adminGlobalActivityExceptions( + accessToken, + startTime?.toISOString().split("T")[0], + endTime?.toISOString().split("T")[0], + modelGroup, + ); + + setGlobalExceptionData(dailyExceptions); + + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( + accessToken, + startTime?.toISOString().split("T")[0], + endTime?.toISOString().split("T")[0], + modelGroup, + ); + + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); + } + } catch (error) { + console.error("Failed to fetch model metrics", error); + } + }; + + const fetchCredentials = async (accessToken: string) => { + try { + const response: CredentialsResponse = await credentialListCall(accessToken); + console.log(`credentials: ${JSON.stringify(response)}`); + setCredentialsList(response.credentials); + } catch (error) { + console.error("Error fetching credentials:", error); + } + }; + + useEffect(() => { + updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to); + }, [selectedAPIKey, selectedCustomer, selectedTeam]); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownOpen(false); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => document.removeEventListener("mousedown", handleClickOutside); + }, []); + + function formatCreatedAt(createdAt: string | null) { + if (createdAt) { + const date = new Date(createdAt); + const options = { month: "long", day: "numeric", year: "numeric" }; + return date.toLocaleDateString("en-US"); + } + return null; + } + + const handleEditClick = (model: any) => { + setSelectedModel(model); + setEditModalVisible(true); + }; + + const handleEditCancel = () => { + setEditModalVisible(false); + setSelectedModel(null); + }; + + const uploadProps: UploadProps = { + name: "file", + accept: ".json", + beforeUpload: (file) => { + if (file.type === "application/json") { + const reader = new FileReader(); + reader.onload = (e) => { + if (e.target) { + const jsonStr = e.target.result as string; + console.log(`Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}`); + addModelForm.setFieldsValue({ vertex_credentials: jsonStr }); + console.log("Form values right after setting:", addModelForm.getFieldsValue()); + } + }; + reader.readAsText(file); + } + // Prevent upload + return false; + }, + onChange(info) { + console.log("Upload onChange triggered with values:", info); + console.log("Current form values:", addModelForm.getFieldsValue()); + + if (info.file.status !== "uploading") { + console.log(info.file, info.fileList); + } + if (info.file.status === "done") { + NotificationsManager.success(`${info.file.name} file uploaded successfully`); + } else if (info.file.status === "error") { + NotificationsManager.fromBackend(`${info.file.name} file upload failed.`); + } + }, + }; + + const handleRefreshClick = () => { + // Update the 'lastRefreshed' state to the current date and time + const currentDate = new Date(); + setLastRefreshed(currentDate.toLocaleString()); + }; + + const handleSaveRetrySettings = async () => { + if (!accessToken) { + console.error("Access token is missing"); + return; + } + + try { + const payload: any = { + router_settings: {}, + }; + + if (selectedModelGroup === "global") { + // Only update global retry policy + console.log("Saving global retry policy:", globalRetryPolicy); + if (globalRetryPolicy) { + payload.router_settings.retry_policy = globalRetryPolicy; + } + NotificationsManager.success("Global retry settings saved successfully"); + } else { + // Only update model group retry policy + console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy); + if (modelGroupRetryPolicy) { + payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy; + } + NotificationsManager.success(`Retry settings saved successfully for ${selectedModelGroup}`); + } + + await setCallbacksCall(accessToken, payload); + } catch (error) { + console.error("Failed to save retry settings:", error); + NotificationsManager.fromBackend("Failed to save retry settings"); + } + }; + + useEffect(() => { + if (!accessToken || !token || !userRole || !userID) { + return; + } + const fetchData = async () => { + try { + // Replace with your actual API call for model data + const modelDataResponse = await modelInfoCall(accessToken, userID, userRole); + console.log("Model data response:", modelDataResponse.data); + setModelData(modelDataResponse); + const _providerSettings = await modelSettingsCall(accessToken); + if (_providerSettings) { + setProviderSettings(_providerSettings); + } + + // loop through modelDataResponse and get all`model_name` values + let all_model_groups: Set = new Set(); + for (let i = 0; i < modelDataResponse.data.length; i++) { + const model = modelDataResponse.data[i]; + all_model_groups.add(model.model_name); + } + console.log("all_model_groups:", all_model_groups); + let _array_model_groups = Array.from(all_model_groups); + // sort _array_model_groups alphabetically + _array_model_groups = _array_model_groups.sort(); + + setAvailableModelGroups(_array_model_groups); + + let all_model_access_groups: Set = new Set(); + for (let i = 0; i < modelDataResponse.data.length; i++) { + const model = modelDataResponse.data[i]; + let model_info: any | null = model.model_info; + if (model_info) { + let access_groups = model_info.access_groups; + if (access_groups) { + for (let j = 0; j < access_groups.length; j++) { + all_model_access_groups.add(access_groups[j]); + } + } + } + } + + setAvailableModelAccessGroups(Array.from(all_model_access_groups)); + + console.log("array_model_groups:", _array_model_groups); + let _initial_model_group = "all"; + if (_array_model_groups.length > 0) { + // set selectedModelGroup to the last model group + _initial_model_group = _array_model_groups[_array_model_groups.length - 1]; + console.log("_initial_model_group:", _initial_model_group); + //setSelectedModelGroup(_initial_model_group); + } + + console.log("selectedModelGroup:", selectedModelGroup); + + const modelMetricsResponse = await modelMetricsCall( + accessToken, + userID, + userRole, + _initial_model_group, + dateValue.from?.toISOString(), + dateValue.to?.toISOString(), + selectedAPIKey?.token, + selectedCustomer, + ); + + console.log("Model metrics response:", modelMetricsResponse); + // Sort by latency (avg_latency_per_token) + + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + + const streamingModelMetricsResponse = await streamingModelMetricsCall( + accessToken, + _initial_model_group, + dateValue.from?.toISOString(), + dateValue.to?.toISOString(), + ); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + setStreamingModelMetrics(streamingModelMetricsResponse.data); + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases); + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + _initial_model_group, + dateValue.from?.toISOString(), + dateValue.to?.toISOString(), + selectedAPIKey?.token, + selectedCustomer, + ); + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + + const slowResponses = await modelMetricsSlowResponsesCall( + accessToken, + userID, + userRole, + _initial_model_group, + dateValue.from?.toISOString(), + dateValue.to?.toISOString(), + selectedAPIKey?.token, + selectedCustomer, + ); + + const dailyExceptions = await adminGlobalActivityExceptions( + accessToken, + dateValue.from?.toISOString().split("T")[0], + dateValue.to?.toISOString().split("T")[0], + _initial_model_group, + ); + + setGlobalExceptionData(dailyExceptions); + + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( + accessToken, + dateValue.from?.toISOString().split("T")[0], + dateValue.to?.toISOString().split("T")[0], + _initial_model_group, + ); + + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); + + console.log("dailyExceptions:", dailyExceptions); + + console.log("dailyExceptionsPerDeplyment:", dailyExceptionsPerDeplyment); + + console.log("slowResponses:", slowResponses); + + setSlowResponsesData(slowResponses); + + let all_end_users_data = await allEndUsersCall(accessToken); + + setAllEndUsers(all_end_users_data?.map((u: any) => u.user_id)); + + const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole); + + let router_settings = routerSettingsInfo.router_settings; + + console.log("routerSettingsInfo:", router_settings); + + let model_group_retry_policy = router_settings.model_group_retry_policy; + let default_retries = router_settings.num_retries; + + console.log("model_group_retry_policy:", model_group_retry_policy); + console.log("default_retries:", default_retries); + setModelGroupRetryPolicy(model_group_retry_policy); + setGlobalRetryPolicy(router_settings.retry_policy); + setDefaultRetry(default_retries); + + // Set model group alias + const model_group_alias = router_settings.model_group_alias || {}; + setModelGroupAlias(model_group_alias); + } catch (error) { + console.error("There was an error fetching the model data", error); + } + }; + + if (accessToken && token && userRole && userID) { + fetchData(); + } + + const fetchModelMap = async () => { + const data = await modelCostMap(accessToken); + console.log(`received model cost map data: ${Object.keys(data)}`); + setModelMap(data); + }; + if (modelMap == null) { + fetchModelMap(); + } + + handleRefreshClick(); + }, [accessToken, token, userRole, userID, modelMap, lastRefreshed, selectedTeam]); + + if (!modelData) { + return
Loading...
; + } + + if (!accessToken || !token || !userRole || !userID) { + return
Loading...
; + } + let all_models_on_proxy: any[] = []; + let all_providers: string[] = []; + + // loop through model data and edit each row + for (let i = 0; i < modelData.data.length; i++) { + let curr_model = modelData.data[i]; + let litellm_model_name = curr_model?.litellm_params?.model; + let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider; + let model_info = curr_model?.model_info; + + let defaultProvider = "openai"; + let provider = ""; + let input_cost = "Undefined"; + let output_cost = "Undefined"; + let max_tokens = "Undefined"; + let max_input_tokens = "Undefined"; + let cleanedLitellmParams = {}; + + const getProviderFromModel = (model: string) => { + /** + * Use model map + * - check if model in model map + * - return it's litellm_provider, if so + */ + console.log(`GET PROVIDER CALLED! - ${modelMap}`); + if (modelMap !== null && modelMap !== undefined) { + if (typeof modelMap == "object" && model in modelMap) { + return modelMap[model]["litellm_provider"]; + } + } + return "openai"; + }; + + // Check if litellm_model_name is null or undefined + if (litellm_model_name) { + // Split litellm_model_name based on "/" + let splitModel = litellm_model_name.split("/"); + + // Get the first element in the split + let firstElement = splitModel[0]; + + // If there is only one element, default provider to openai + provider = custom_llm_provider; + if (!provider) { + provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement; + } + } else { + // litellm_model_name is null or undefined, default provider to openai + provider = "-"; + } + + if (model_info) { + input_cost = model_info?.input_cost_per_token; + output_cost = model_info?.output_cost_per_token; + max_tokens = model_info?.max_tokens; + max_input_tokens = model_info?.max_input_tokens; + } + + if (curr_model?.litellm_params) { + cleanedLitellmParams = Object.fromEntries( + Object.entries(curr_model?.litellm_params).filter(([key]) => key !== "model" && key !== "api_base"), + ); + } + + modelData.data[i].provider = provider; + modelData.data[i].input_cost = input_cost; + modelData.data[i].output_cost = output_cost; + modelData.data[i].litellm_model_name = litellm_model_name; + all_providers.push(provider); + + // Convert Cost in terms of Cost per 1M tokens + if (modelData.data[i].input_cost) { + modelData.data[i].input_cost = (Number(modelData.data[i].input_cost) * 1000000).toFixed(2); + } + + if (modelData.data[i].output_cost) { + modelData.data[i].output_cost = (Number(modelData.data[i].output_cost) * 1000000).toFixed(2); + } + + modelData.data[i].max_tokens = max_tokens; + modelData.data[i].max_input_tokens = max_input_tokens; + modelData.data[i].api_base = curr_model?.litellm_params?.api_base; + modelData.data[i].cleanedLitellmParams = cleanedLitellmParams; + + all_models_on_proxy.push(curr_model.model_name); + + console.log(modelData.data[i]); + } + // when users click request access show pop up to allow them to request access + + if (userRole && userRole == "Admin Viewer") { + const { Title, Paragraph } = Typography; + return ( +
+ Access Denied + Ask your proxy admin for access to view all models +
+ ); + } + + const runHealthCheck = async () => { + try { + NotificationsManager.info("Running health check..."); + setIsHealthCheckLoading(true); + setHealthCheckResponse(null); + const response = await healthCheckCall(accessToken); + setHealthCheckResponse(response); + } catch (error) { + console.error("Error running health check:", error); + setHealthCheckResponse("Error running health check"); + } finally { + setIsHealthCheckLoading(false); + } + }; + + const FilterByContent = ( +
+ Select API Key Name + + {premiumUser ? ( +
+ + + Select Customer Name + + + + Select Team + + +
+ ) : ( +
+ {/* ... existing non-premium user content ... */} + Select Team + + +
+ )} +
+ ); + + const customTooltip = (props: any) => { + const { payload, active } = props; + if (!active || !payload) return null; + + // Extract the date from the first item in the payload array + const date = payload[0]?.payload?.date; + + // Sort the payload array by category.value in descending order + let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); + + // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown + if (sortedPayload.length > 5) { + let remainingItems = sortedPayload.length - 5; + sortedPayload = sortedPayload.slice(0, 5); + sortedPayload.push({ + dataKey: `${remainingItems} other deployments`, + value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), + color: "gray", + }); + } + + return ( +
+ {date &&

Date: {date}

} + {sortedPayload.map((category: any, idx: number) => { + const roundedValue = parseFloat(category.value.toFixed(5)); + const displayValue = roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5); + return ( +
+
+
+

{category.dataKey}

+
+

{displayValue}

+
+ ); + })} +
+ ); + }; + + const handleOk = () => { + console.log("🚀 handleOk called from model dashboard!"); + console.log("Current form values:", addModelForm.getFieldsValue()); + + addModelForm + .validateFields() + .then((values: any) => { + console.log("✅ Validation passed, submitting:", values); + handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick); + }) + .catch((error: any) => { + console.error("❌ Validation failed:", error); + console.error("Form errors:", error.errorFields); + const errorMessages = + error.errorFields + ?.map((field: any) => { + return `${field.name.join(".")}: ${field.errors.join(", ")}`; + }) + .join(" | ") || "Unknown validation error"; + NotificationsManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`); + }); + }; + + console.log(`selectedProvider: ${selectedProvider}`); + console.log(`providerModels.length: ${providerModels.length}`); + + const providerKey = Object.keys(Providers).find( + (key) => (Providers as { [index: string]: any })[key] === selectedProvider, + ); + + let dynamicProviderForm: ProviderSettings | undefined = undefined; + if (providerKey && providerSettings) { + dynamicProviderForm = providerSettings.find((provider) => provider.name === provider_map[providerKey]); + } + + // If a team is selected, render TeamInfoView in full page layout + if (selectedTeamId) { + return ( +
+ setSelectedTeamId(null)} + accessToken={accessToken} + is_team_admin={userRole === "Admin"} + is_proxy_admin={userRole === "Proxy Admin"} + userModels={all_models_on_proxy} + editTeam={false} + onUpdate={handleRefreshClick} + /> +
+ ); + } + + return ( +
+ + + {/* Model Management Header */} +
+
+

Model Management

+ {!all_admin_roles.includes(userRole) ? ( +

Add models for teams you are an admin for.

+ ) : ( +

Add and manage models for the proxy

+ )} +
+
+ {selectedModelId ? ( + { + setSelectedModelId(null); + setEditModel(false); + }} + modelData={modelData.data.find((model: any) => model.model_info.id === selectedModelId)} + accessToken={accessToken} + userID={userID} + userRole={userRole} + setEditModalVisible={setEditModalVisible} + setSelectedModel={setSelectedModel} + onModelUpdate={(updatedModel) => { + // Update the model in the modelData.data array + const updatedModelData = { + ...modelData, + data: modelData.data.map((model: any) => + model.model_info.id === updatedModel.model_info.id ? updatedModel : model, + ), + }; + setModelData(updatedModelData); + // Trigger a refresh to update UI + handleRefreshClick(); + }} + modelAccessGroups={availableModelAccessGroups} + /> + ) : ( + + +
+ {all_admin_roles.includes(userRole) ? All Models : Your Models} + Add Model + {all_admin_roles.includes(userRole) && LLM Credentials} + {all_admin_roles.includes(userRole) && Pass-Through Endpoints} + {all_admin_roles.includes(userRole) && Health Status} + {all_admin_roles.includes(userRole) && Model Analytics} + {all_admin_roles.includes(userRole) && Model Retry Settings} + {all_admin_roles.includes(userRole) && Model Group Alias} + {all_admin_roles.includes(userRole) && Price Data Reload} +
+ +
+ {lastRefreshed && Last Refreshed: {lastRefreshed}} + +
+ + + + +
+
+ {/* Current Team and View Mode Selector - Prominent Section */} +
+
+
+ Current Team: + +
+ +
+ View: + +
+
+ + {modelViewMode === "current_team" && ( +
+ +
+ {currentTeam === "personal" ? ( + + To access these models: Create a Virtual Key without selecting a team on the{" "} + + Virtual Keys page + + + ) : ( + + To access these models: Create a Virtual Key and select Team as " + {currentTeam}" on the{" "} + + Virtual Keys page + + + )} +
+
+ )} +
+ + {/* Search and Filter Controls */} +
+
+ {/* Search and Filter Controls */} +
+ {/* Model Name Search */} +
+ setModelNameSearch(e.target.value)} + /> + + + +
+ + {/* Filter Button */} + + + {/* Reset Filters Button */} + +
+ + {/* Additional Filters */} + {showFilters && ( +
+ {/* Model Name Filter */} +
+ +
+ + {/* Model Access Group Filter */} +
+ +
+
+ )} + + {/* Results Count and Pagination Controls */} +
+ + {filteredData.length > 0 + ? `Showing ${pagination.pageIndex * pagination.pageSize + 1} - ${Math.min( + (pagination.pageIndex + 1) * pagination.pageSize, + filteredData.length, + )} of ${filteredData.length} results` + : "Showing 0 results"} + + + {/* Pagination Controls */} + {filteredData.length > pagination.pageSize && ( +
+ + + +
+ )} +
+
+
+ + +
+
+ + + + + + + + + + + + + + + + + + { + setDateValue(value); + updateModelMetrics(selectedModelGroup, value.from, value.to); + }} + /> + + + Select Model Group + + + + + + + + + + + + + + + Avg. Latency per Token + Time to first token + + + +

(seconds/token)

+ + average Latency for successfull requests divided by the total tokens + + {modelMetrics && modelMetricsCategories && ( + + )} + + + + + + + + + + + + + + Deployment + Success Responses + + Slow Responses

Success Responses taking 600+s

+ + + + + {slowResponsesData.map((metric, idx) => ( + + {metric.api_base} + {metric.total_count} + {metric.slow_count} + + ))} + +
+ + + + + + All Exceptions for {selectedModelGroup} + + + + + + + + All Up Rate Limit Errors (429) for {selectedModelGroup} + + + + Num Rate Limit Errors {globalExceptionData.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + + + + + {premiumUser ? ( + <> + {globalExceptionPerDeployment.map((globalActivity, index) => ( + + {globalActivity.api_base ? globalActivity.api_base : "Unknown API Base"} + + + + Num Rate Limit Errors (429) {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + + + ))} + + ) : ( + <> + {globalExceptionPerDeployment && + globalExceptionPerDeployment.length > 0 && + globalExceptionPerDeployment.slice(0, 1).map((globalActivity, index) => ( + + ✨ Rate Limit Errors by Deployment +

+ Upgrade to see exceptions for all deployments +

+ + + {globalActivity.api_base} + + + + Num Rate Limit Errors {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + + + + ))} + + )} + + + +
+
+ Retry Policy Scope: + +
+
+ + {selectedModelGroup === "global" ? ( + <> + Global Retry Policy + Default retry settings applied to all model groups unless overridden + + ) : ( + <> + Retry Policy for {selectedModelGroup} + + Model-specific retry settings. Falls back to global defaults if not set. + + + )} + {retry_policy_map && ( + + + {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => { + let retryCount: number; + + if (selectedModelGroup === "global") { + // Show global policy values + retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; + } else { + // Show model-group specific values with fallback to global + const modelSpecificCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey]; + if (modelSpecificCount != null) { + retryCount = modelSpecificCount; + } else { + // Fall back to global policy, then default + retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; + } + } + + return ( + + + + + ); + })} + +
+ {exceptionType} + {selectedModelGroup !== "global" && ( + + (Global: {globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry}) + + )} + + { + if (selectedModelGroup === "global") { + // Update global policy + setGlobalRetryPolicy((prevGlobalRetryPolicy) => { + if (value == null) return prevGlobalRetryPolicy; + return { + ...(prevGlobalRetryPolicy ?? {}), + [retryPolicyKey]: value, + }; + }); + } else { + // Update model-group specific policy + setModelGroupRetryPolicy((prevModelGroupRetryPolicy) => { + const prevRetryPolicy = prevModelGroupRetryPolicy?.[selectedModelGroup!] ?? {}; + return { + ...(prevModelGroupRetryPolicy ?? {}), + [selectedModelGroup!]: { + ...prevRetryPolicy, + [retryPolicyKey!]: value, + }, + } as RetryPolicyObject; + }); + } + }} + /> +
+ )} + + + + + + +
+
+ Price Data Management + + Manage model pricing data and configure automatic reload schedules + +
+ { + // Refresh the model map after successful reload + const fetchModelMap = async () => { + const data = await modelCostMap(accessToken); + setModelMap(data); + }; + fetchModelMap(); + }} + buttonText="Reload Price Data" + size="middle" + type="primary" + className="w-full" + /> +
+ + + + )} + + +
+ ); +}; + +export default OldModelDashboard;