From a93471a3c79dd4ef058ebb59c55e0ecaca68c168 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 16:54:24 -0700 Subject: [PATCH 1/4] feat(team_endpoints.py): unfurl 'all-proxy-models' on team info endpoints --- litellm/proxy/_new_secret_config.yaml | 4 ++- .../management_endpoints/team_endpoints.py | 31 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 76317f050f..9918ce429e 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -4,4 +4,6 @@ model_list: model: azure/chatgpt-v-2 api_key: os.environ/AZURE_API_KEY api_base: os.environ/AZURE_API_BASE - \ No newline at end of file + +litellm_settings: + callbacks: ["prometheus"] \ No newline at end of file diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 25b42e7e02..81f66421a7 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -40,6 +40,7 @@ from litellm.proxy._types import ( ProxyErrorTypes, ProxyException, SpecialManagementEndpointEnums, + SpecialModelNames, TeamAddMemberResponse, TeamInfoResponseObject, TeamListResponseObject, @@ -70,6 +71,7 @@ from litellm.proxy.utils import ( _premium_user_check, handle_exception_on_proxy, ) +from litellm.router import Router router = APIRouter() @@ -1238,6 +1240,23 @@ def validate_membership( ) +def _unfurl_all_proxy_models( + team_info: LiteLLM_TeamTable, llm_router: Router +) -> LiteLLM_TeamTable: + if ( + SpecialModelNames.all_proxy_models.value in team_info.models + and llm_router is not None + ): + team_models: set[str] = set() # make set to avoid duplicates + for model in team_info.models: + if model != SpecialModelNames.all_proxy_models.value: + team_models.add(model) + for model in llm_router.get_model_names(): + team_models.add(model) + team_info.models = list(team_models) + return team_info + + @router.get( "/team/info", tags=["team management"], dependencies=[Depends(user_api_key_auth)] ) @@ -1260,7 +1279,7 @@ async def team_info( --header 'Authorization: Bearer your_api_key_here' ``` """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import llm_router, prisma_client try: if prisma_client is None: @@ -1333,6 +1352,9 @@ async def team_info( else: _team_info = LiteLLM_TeamTable() + ## UNFURL 'all-proxy-models' into the team_info.models list ## + if llm_router is not None: + _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, @@ -1534,7 +1556,7 @@ async def list_team( - user_id: str - Optional. If passed will only return teams that the user_id is a member of. - organization_id: str - Optional. If passed will only return teams that belong to the organization_id. Pass 'default_organization' to get all teams without organization_id. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import llm_router, prisma_client if not allowed_route_check_inside_route( user_api_key_dict=user_api_key_dict, requested_user_id=user_id @@ -1593,6 +1615,11 @@ async def list_team( ) try: + # unfurl all-proxy-models + if llm_router is not None: + team = _unfurl_all_proxy_models( + LiteLLM_TeamTable(**team.model_dump()), llm_router + ) returned_responses.append( TeamListResponseObject( **team.model_dump(), From e914a9ddf11f0fc9439eb9d2f81df562867afe06 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 17:44:45 -0700 Subject: [PATCH 2/4] fix(model_info_view.tsx): call db for model info --- .../src/components/model_info_view.tsx | 59 +++++++++++-------- .../src/components/networking.tsx | 30 ++++++++++ 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 7768aa71cb..d8d1ded81b 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -15,7 +15,7 @@ import { NumberInput, } from "@tremor/react"; import { ArrowLeftIcon, TrashIcon, KeyIcon } from "@heroicons/react/outline"; -import { modelDeleteCall, modelUpdateCall, CredentialItem, credentialGetCall, credentialCreateCall } from "./networking"; +import { modelDeleteCall, modelUpdateCall, CredentialItem, credentialGetCall, credentialCreateCall, modelInfoCall, modelInfoV1Call } from "./networking"; import { Button, Form, Input, InputNumber, message, Select, Modal } from "antd"; import EditModelModal from "./edit_model/edit_model_modal"; import { handleEditModelSubmit } from "./edit_model/edit_model_modal"; @@ -48,7 +48,7 @@ export default function ModelInfoView({ setSelectedModel }: ModelInfoViewProps) { const [form] = Form.useForm(); - const [localModelData, setLocalModelData] = useState(modelData); + const [localModelData, setLocalModelData] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [isCredentialModalOpen, setIsCredentialModalOpen] = useState(false); const [isDirty, setIsDirty] = useState(false); @@ -77,7 +77,16 @@ export default function ModelInfoView({ credential_info: existingCredentialResponse["credential_info"] }); } + + const getModelInfo = async () => { + if (!accessToken) return; + let modelInfoResponse = await modelInfoV1Call(accessToken, modelId); + console.log("modelInfoResponse, ", modelInfoResponse); + let specificModelData = modelInfoResponse.data[0]; + setLocalModelData(specificModelData); + } getExistingCredential(); + getModelInfo(); }, [accessToken, modelId]); const handleReuseCredential = async (values: any) => { @@ -292,24 +301,25 @@ export default function ModelInfoView({ )} -
setIsDirty(true)} @@ -346,9 +356,9 @@ export default function ModelInfoView({ ) : (
- {localModelData.litellm_params?.input_cost_per_token - ? (localModelData.litellm_params.input_cost_per_token * 1_000_000).toFixed(4) - : modelData.input_cost * 1_000_000} + {localModelData?.litellm_params?.input_cost_per_token + ? (localModelData.litellm_params?.input_cost_per_token * 1_000_000).toFixed(4) + : localModelData?.model_info?.input_cost_per_token ? (localModelData.model_info.input_cost_per_token * 1_000_000).toFixed(4) : null}
)} @@ -361,9 +371,9 @@ export default function ModelInfoView({ ) : (
- {localModelData.litellm_params?.output_cost_per_token + {localModelData?.litellm_params?.output_cost_per_token ? (localModelData.litellm_params.output_cost_per_token * 1_000_000).toFixed(4) - : modelData.output_cost * 1_000_000} + : localModelData?.model_info?.output_cost_per_token ? (localModelData.model_info.output_cost_per_token * 1_000_000).toFixed(4) : null}
)} @@ -502,7 +512,10 @@ export default function ModelInfoView({ )} -
+ + ) : ( + Loading... + )} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 0c1ba991ac..831ff7d102 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1254,6 +1254,36 @@ export const modelInfoCall = async ( } }; +export const modelInfoV1Call = async (accessToken: String, modelId: String) => { + /** + * Get all models on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`; + url += `?litellm_model_id=${modelId}`; + + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log("modelInfoV1Call:", data); + return data; + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + export const modelHubCall = async (accessToken: String) => { /** * Get all models on proxy From 9a670552fd50d6b25847c31748196509764523ec Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 17:52:36 -0700 Subject: [PATCH 3/4] refactor(transform_request/): remove experimental transform request tab from ui move to api docs instead --- ui/litellm-dashboard/src/app/page.tsx | 4 +--- ui/litellm-dashboard/src/components/leftnav.tsx | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index e796444514..876a6b7c1c 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -310,9 +310,7 @@ export default function CreateKeyPage() { ) : page == "guardrails" ? ( - ) : page == "transform-request" ? ( - - ) : page == "general-settings" ? ( + ): page == "general-settings" ? ( , roles: all_admin_roles }, { key: "10", page: "budgets", label: "Budgets", icon: , roles: all_admin_roles }, { key: "11", page: "guardrails", label: "Guardrails", icon: , roles: all_admin_roles }, - { key: "18", page: "transform-request", label: "Playground", icon: , roles: all_admin_roles }, ] }, { From 50109a7784f00a3e1ac2c8f6a68ae6db34354f6f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 17:59:40 -0700 Subject: [PATCH 4/4] fix(top_key_view.tsx): fix bar chart to use key alias --- .../src/components/top_key_view.tsx | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/top_key_view.tsx b/ui/litellm-dashboard/src/components/top_key_view.tsx index d687e499d3..d6643721ad 100644 --- a/ui/litellm-dashboard/src/components/top_key_view.tsx +++ b/ui/litellm-dashboard/src/components/top_key_view.tsx @@ -122,7 +122,7 @@ const TopKeyView: React.FC = ({ = ({ layout="vertical" showXAxis={false} showLegend={false} - valueFormatter={(value) => `$${value.toFixed(2)}`} + valueFormatter={(value) => value ? `$${value.toFixed(2)}` : "No Key Alias"} onValueChange={(item) => handleKeyClick(item)} showTooltip={true} + customTooltip={(props) => { + const item = props.payload?.[0]?.payload; + return ( +
+
+
+ Key: + {item?.key?.slice(0, 10)}... +
+
+ Spend: + ${item?.spend.toFixed(2)} +
+
+
+ ); + }} /> ) : (