From f7f1ce9345b93df6e49a0b3cfe16935d80f1a13a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:19:12 -0700 Subject: [PATCH 01/11] feat - get all unique end_users --- litellm/proxy/proxy_server.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 266c893521..c6ea505f86 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8917,6 +8917,38 @@ async def global_spend_per_team(): } +@router.get( + "/global/all_end_users", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) +async def global_view_all_end_users(): + """ + [BETA] This is a beta endpoint. It will change. + + Use this to just get all the unique `end_users` + """ + global prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + sql_query = """ + SELECT DISTINCT end_user FROM "LiteLLM_SpendLogs" + """ + + db_response = await prisma_client.db.query_raw(query=sql_query) + if db_response is None: + return [] + + _end_users = [] + for row in db_response: + _end_users.append(row["end_user"]) + + return {"end_users": _end_users} + + @router.post( "/global/spend/end_users", tags=["Budget & Spend Tracking"], From 7a1610327fa0ac5cd1cadad7ac46dc762e5a6050 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:26:09 -0700 Subject: [PATCH 02/11] ui - view all end-users on admin ui --- .../src/components/model_dashboard.tsx | 29 +++++++++++++++++ .../src/components/networking.tsx | 32 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index ea23f660a5..050df55299 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -51,6 +51,7 @@ import { modelSettingsCall, adminGlobalActivityExceptions, adminGlobalActivityExceptionsPerDeployment, + allEndUsersCall, } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { @@ -320,6 +321,8 @@ const ModelDashboard: React.FC = ({ const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); const [selectedAPIKey, setSelectedAPIKey] = useState(null); + const [allEndUsers, setAllEndUsers] = useState([]); + useEffect(() => { updateModelMetrics( selectedModelGroup, @@ -700,6 +703,10 @@ const ModelDashboard: React.FC = ({ setSlowResponsesData(slowResponses); + let all_end_users_data = await allEndUsersCall(accessToken); + + setAllEndUsers(all_end_users_data?.end_users); + const routerSettingsInfo = await getCallbacksCall( accessToken, userID, @@ -1033,6 +1040,28 @@ const ModelDashboard: React.FC = ({ return null; // Add this line to handle the case when the condition is not met })} + + + + Select Customer Name + + diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 601f2c432b..021322bf52 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1053,6 +1053,38 @@ export const allTagNamesCall = async ( }; +export const allEndUsersCall = async ( + accessToken: String, +) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/global/all_end_users` + : `/global/all_end_users`; + + + console.log("in global/all_end_users call", url); + const response = await fetch(`${url}`, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const errorData = await response.text(); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log(data); + return data; + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + + export const userSpendLogsCall = async ( accessToken: String, token: String, From 856018d5dfcb9461bdd389978554f0f1a3e4d5ed Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:30:06 -0700 Subject: [PATCH 03/11] ui - filter by all customers --- .../src/components/model_dashboard.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 050df55299..82836badcd 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1045,7 +1045,17 @@ const ModelDashboard: React.FC = ({ Select Customer Name - + { + setSelectedAPIKey(null); + }} + > + All Customers + { allEndUsers?.map((user: any, index: number) => { return ( From 9e6256b688cda88172b9058874752d45688b5635 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:41:31 -0700 Subject: [PATCH 04/11] ui - filter analytics by customer --- .../src/components/model_dashboard.tsx | 25 +++++++++++++------ .../src/components/networking.tsx | 13 ++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 82836badcd..f9c172f1ae 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -320,6 +320,7 @@ const ModelDashboard: React.FC = ({ const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); const [selectedAPIKey, setSelectedAPIKey] = useState(null); + const [selectedCustomer, setSelectedCustomer] = useState(null); const [allEndUsers, setAllEndUsers] = useState([]); @@ -329,7 +330,7 @@ const ModelDashboard: React.FC = ({ dateValue.from, dateValue.to ); - }, [selectedAPIKey]); + }, [selectedAPIKey, selectedCustomer]); function formatCreatedAt(createdAt: string | null) { if (createdAt) { @@ -918,6 +919,11 @@ const ModelDashboard: React.FC = ({ selected_token = null; } + let selected_customer = selectedCustomer; + if (selected_customer === undefined) { + selected_customer = null; + } + try { const modelMetricsResponse = await modelMetricsCall( accessToken, @@ -926,7 +932,8 @@ const ModelDashboard: React.FC = ({ modelGroup, startTime.toISOString(), endTime.toISOString(), - selected_token + selected_token, + selected_customer ); console.log("Model metrics response:", modelMetricsResponse); @@ -954,7 +961,8 @@ const ModelDashboard: React.FC = ({ modelGroup, startTime.toISOString(), endTime.toISOString(), - selected_token + selected_token, + selected_customer ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); @@ -967,7 +975,8 @@ const ModelDashboard: React.FC = ({ modelGroup, startTime.toISOString(), endTime.toISOString(), - selected_token + selected_token, + selected_customer ); console.log("slowResponses:", slowResponses); @@ -1051,7 +1060,7 @@ const ModelDashboard: React.FC = ({ key="all-customers" value="all-customers" onClick={() => { - setSelectedAPIKey(null); + setSelectedCustomer(null); }} > All Customers @@ -1062,9 +1071,9 @@ const ModelDashboard: React.FC = ({ { - // setSelectedEndUser(user); - // }} + onClick={() => { + setSelectedCustomer(user); + }} > {user} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 021322bf52..2b89c5ef1d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -728,6 +728,7 @@ export const modelMetricsCall = async ( startTime: String | undefined, endTime: String | undefined, apiKey: String | null, + customer: String | null ) => { /** * Get all models on proxy @@ -735,7 +736,7 @@ export const modelMetricsCall = async ( try { let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`; if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`; + url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; } // message.info("Requesting model data"); const response = await fetch(url, { @@ -807,7 +808,8 @@ export const modelMetricsSlowResponsesCall = async ( modelGroup: String | null, startTime: String | undefined, endTime: String | undefined, - apiKey: String | null + apiKey: String | null, + customer: String | null ) => { /** * Get all models on proxy @@ -817,7 +819,7 @@ export const modelMetricsSlowResponsesCall = async ( ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`; if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`; + url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; } // message.info("Requesting model data"); @@ -851,7 +853,8 @@ export const modelExceptionsCall = async ( modelGroup: String | null, startTime: String | undefined, endTime: String | undefined, - apiKey: String | null + apiKey: String | null, + customer: String | null ) => { /** * Get all models on proxy @@ -862,7 +865,7 @@ export const modelExceptionsCall = async ( : `/model/metrics/exceptions`; if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`; + url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; } const response = await fetch(url, { method: "GET", From 016b8294b01b9cbe149d7056fe1ea54042e73c1a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:42:17 -0700 Subject: [PATCH 05/11] feat - filter analytics by customer --- litellm/proxy/proxy_server.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c6ea505f86..ba51eaeb41 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11648,6 +11648,7 @@ async def model_metrics( startTime: Optional[datetime] = None, endTime: Optional[datetime] = None, api_key: Optional[str] = None, + customer: Optional[str] = None, ): global prisma_client, llm_router if prisma_client is None: @@ -11663,6 +11664,9 @@ async def model_metrics( if api_key is None or api_key == "undefined": api_key = "null" + if customer is None or customer == "undefined": + customer = "null" + sql_query = """ SELECT api_base, @@ -11681,6 +11685,12 @@ async def model_metrics( ELSE TRUE END ) + AND ( + CASE + WHEN $5 != 'null' THEN "end_user" = $5 + ELSE TRUE + END + ) GROUP BY api_base, model_group, @@ -11693,7 +11703,7 @@ async def model_metrics( """ _all_api_bases = set() db_response = await prisma_client.db.query_raw( - sql_query, _selected_model_group, startTime, endTime, api_key + sql_query, _selected_model_group, startTime, endTime, api_key, customer ) _daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}} @@ -11753,6 +11763,7 @@ async def model_metrics_slow_responses( startTime: Optional[datetime] = None, endTime: Optional[datetime] = None, api_key: Optional[str] = None, + customer: Optional[str] = None, ): global prisma_client, llm_router, proxy_logging_obj if prisma_client is None: @@ -11765,6 +11776,9 @@ async def model_metrics_slow_responses( if api_key is None or api_key == "undefined": api_key = "null" + if customer is None or customer == "undefined": + customer = "null" + startTime = startTime or datetime.now() - timedelta(days=30) endTime = endTime or datetime.now() @@ -11794,6 +11808,12 @@ WHERE ELSE TRUE END ) + AND ( + CASE + WHEN $6 != 'null' THEN "end_user" = $6 + ELSE TRUE + END + ) GROUP BY api_base ORDER BY @@ -11807,6 +11827,7 @@ ORDER BY startTime, endTime, api_key, + customer, ) if db_response is not None: @@ -11831,6 +11852,7 @@ async def model_metrics_exceptions( startTime: Optional[datetime] = None, endTime: Optional[datetime] = None, api_key: Optional[str] = None, + customer: Optional[str] = None, ): global prisma_client, llm_router if prisma_client is None: From 0c808c8afa4d4fe022abf32524cd3f0f8eb3c91f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:49:05 -0700 Subject: [PATCH 06/11] fix - analytics use last hour of endTime and first hr startTime --- .../src/components/model_dashboard.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index f9c172f1ae..a2c1a28072 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -924,6 +924,16 @@ const ModelDashboard: React.FC = ({ selected_customer = null; } + // make startTime and endTime to last hour of the day + startTime.setHours(0); + startTime.setMinutes(0); + startTime.setSeconds(0); + + endTime.setHours(23); + endTime.setMinutes(59); + endTime.setSeconds(59); + + try { const modelMetricsResponse = await modelMetricsCall( accessToken, From c1bac5e89dd17c2986a47fa367a2ffa8902c08b6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:49:21 -0700 Subject: [PATCH 07/11] feat - use inlcusive analytics on proxy --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ba51eaeb41..c154bd8e97 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11677,7 +11677,7 @@ async def model_metrics( FROM "LiteLLM_SpendLogs" WHERE - "startTime" BETWEEN $2::timestamp AND $3::timestamp + "startTime" >= $2::timestamp AND "startTime" <= $3::timestamp AND "model_group" = $1 AND "cache_hit" != 'True' AND ( CASE From 3a6d5a2dfb1fb0e799e2b677d28d40c8faf0bd6c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:55:04 -0700 Subject: [PATCH 08/11] feat enforce viweing analytivs by service behing enterprise --- .../src/components/model_dashboard.tsx | 85 ++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index a2c1a28072..57c78b6c21 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1025,9 +1025,11 @@ const ModelDashboard: React.FC = ({ const FilterByContent = (
Select API Key Name - - = ({ } +
+ ): ( +
+ + + + + + Select Customer Name + + + + +
+ ) + } + + + From 844ae1c1eb6e95b658c21e149a3aeae0fb777a68 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 20:59:36 -0700 Subject: [PATCH 09/11] fix selectedCustomer --- .../src/components/model_dashboard.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 57c78b6c21..0484d15b58 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -632,7 +632,8 @@ const ModelDashboard: React.FC = ({ _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), - selectedAPIKey?.token + selectedAPIKey?.token, + selectedCustomer ); console.log("Model metrics response:", modelMetricsResponse); @@ -661,7 +662,8 @@ const ModelDashboard: React.FC = ({ _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), - selectedAPIKey?.token + selectedAPIKey?.token, + selectedCustomer ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); @@ -674,7 +676,8 @@ const ModelDashboard: React.FC = ({ _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), - selectedAPIKey?.token + selectedAPIKey?.token, + selectedCustomer ); const dailyExceptions = await adminGlobalActivityExceptions( @@ -1977,7 +1980,6 @@ const ModelDashboard: React.FC = ({