tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-08 00:21:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3997 from BerriAI/litellm_filter_latency_by_customer

Browse Source 
[Admin UI] Filter Model Latency by Customer, API Key
This commit is contained in:
Ishaan Jaff
2024-06-03 21:06:25 -07:00
committed by GitHub
parent 5ee3b0f30f 0a316dbea3
commit 0fa370ae44
3 changed files with 253 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/proxy/proxy_server.py
+56 -2
View File
@@ -8917,6 +8917,38 @@ async def global_spend_per_team():
}
@router.get(
"/global/all_end_users",
tags=["Budget & Spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
include_in_schema=False,
)
async def global_view_all_end_users():
"""
[BETA] This is a beta endpoint. It will change.
Use this to just get all the unique `end_users`
"""
global prisma_client
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
sql_query = """
SELECT DISTINCT end_user FROM "LiteLLM_SpendLogs"
"""
db_response = await prisma_client.db.query_raw(query=sql_query)
if db_response is None:
return []
_end_users = []
for row in db_response:
_end_users.append(row["end_user"])
return {"end_users": _end_users}
@router.post(
"/global/spend/end_users",
tags=["Budget & Spend Tracking"],
@@ -11616,6 +11648,7 @@ async def model_metrics(
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
customer: Optional[str] = None,
):
global prisma_client, llm_router
if prisma_client is None:
@@ -11631,6 +11664,9 @@ async def model_metrics(
if api_key is None or api_key == "undefined":
api_key = "null"
if customer is None or customer == "undefined":
customer = "null"
sql_query = """
SELECT
api_base,
@@ -11641,7 +11677,7 @@ async def model_metrics(
FROM
"LiteLLM_SpendLogs"
WHERE
"startTime" BETWEEN $2::timestamp AND $3::timestamp
"startTime" >= $2::timestamp AND "startTime" <= $3::timestamp
AND "model_group" = $1 AND "cache_hit" != 'True'
AND (
CASE
@@ -11649,6 +11685,12 @@ async def model_metrics(
ELSE TRUE
END
)
AND (
CASE
WHEN $5 != 'null' THEN "end_user" = $5
ELSE TRUE
END
)
GROUP BY
api_base,
model_group,
@@ -11661,7 +11703,7 @@ async def model_metrics(
"""
_all_api_bases = set()
db_response = await prisma_client.db.query_raw(
sql_query, _selected_model_group, startTime, endTime, api_key
sql_query, _selected_model_group, startTime, endTime, api_key, customer
)
_daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}}
@@ -11721,6 +11763,7 @@ async def model_metrics_slow_responses(
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
customer: Optional[str] = None,
):
global prisma_client, llm_router, proxy_logging_obj
if prisma_client is None:
@@ -11733,6 +11776,9 @@ async def model_metrics_slow_responses(
if api_key is None or api_key == "undefined":
api_key = "null"
if customer is None or customer == "undefined":
customer = "null"
startTime = startTime or datetime.now() - timedelta(days=30)
endTime = endTime or datetime.now()
@@ -11762,6 +11808,12 @@ WHERE
ELSE TRUE
END
)
AND (
CASE
WHEN $6 != 'null' THEN "end_user" = $6
ELSE TRUE
END
)
GROUP BY
api_base
ORDER BY
@@ -11775,6 +11827,7 @@ ORDER BY
startTime,
endTime,
api_key,
customer,
)
if db_response is not None:
@@ -11799,6 +11852,7 @@ async def model_metrics_exceptions(
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
customer: Optional[str] = None,
):
global prisma_client, llm_router
if prisma_client is None:
ui/litellm-dashboard/src/components/model_dashboard.tsx
+157 -12
View File
@@ -51,6 +51,7 @@ import {
modelSettingsCall,
adminGlobalActivityExceptions,
adminGlobalActivityExceptionsPerDeployment,
allEndUsersCall,
} from "./networking";
import { BarChart, AreaChart } from "@tremor/react";
import {
@@ -319,6 +320,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [showAdvancedFilters, setShowAdvancedFilters] = useState<boolean>(false);
const [selectedAPIKey, setSelectedAPIKey] = useState<any | null>(null);
const [selectedCustomer, setSelectedCustomer] = useState<any | null>(null);
const [allEndUsers, setAllEndUsers] = useState<any[]>([]);
useEffect(() => {
updateModelMetrics(
@@ -326,7 +330,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
dateValue.from,
dateValue.to
);
}, [selectedAPIKey]);
}, [selectedAPIKey, selectedCustomer]);
function formatCreatedAt(createdAt: string | null) {
if (createdAt) {
@@ -628,7 +632,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString(),
selectedAPIKey?.token
selectedAPIKey?.token,
selectedCustomer
);
console.log("Model metrics response:", modelMetricsResponse);
@@ -657,7 +662,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString(),
selectedAPIKey?.token
selectedAPIKey?.token,
selectedCustomer
);
console.log("Model exceptions response:", modelExceptionsResponse);
setModelExceptions(modelExceptionsResponse.data);
@@ -670,7 +676,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString(),
selectedAPIKey?.token
selectedAPIKey?.token,
selectedCustomer
);
const dailyExceptions = await adminGlobalActivityExceptions(
@@ -700,6 +707,10 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
setSlowResponsesData(slowResponses);
let all_end_users_data = await allEndUsersCall(accessToken);
setAllEndUsers(all_end_users_data?.end_users);
const routerSettingsInfo = await getCallbacksCall(
accessToken,
userID,
@@ -911,6 +922,21 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
selected_token = null;
}
let selected_customer = selectedCustomer;
if (selected_customer === undefined) {
selected_customer = null;
}
// make startTime and endTime to last hour of the day
startTime.setHours(0);
startTime.setMinutes(0);
startTime.setSeconds(0);
endTime.setHours(23);
endTime.setMinutes(59);
endTime.setSeconds(59);
try {
const modelMetricsResponse = await modelMetricsCall(
accessToken,
@@ -919,7 +945,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
modelGroup,
startTime.toISOString(),
endTime.toISOString(),
selected_token
selected_token,
selected_customer
);
console.log("Model metrics response:", modelMetricsResponse);
@@ -947,7 +974,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
modelGroup,
startTime.toISOString(),
endTime.toISOString(),
selected_token
selected_token,
selected_customer
);
console.log("Model exceptions response:", modelExceptionsResponse);
setModelExceptions(modelExceptionsResponse.data);
@@ -960,7 +988,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
modelGroup,
startTime.toISOString(),
endTime.toISOString(),
selected_token
selected_token,
selected_customer
);
console.log("slowResponses:", slowResponses);
@@ -999,9 +1028,11 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const FilterByContent = (
<div >
<Text className="mb-1">Select API Key Name</Text>
<Select defaultValue="all-keys">
{
premiumUser ? (
<div>
<Select defaultValue="all-keys">
<SelectItem
key="all-keys"
value="all-keys"
@@ -1033,7 +1064,118 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
return null; // Add this line to handle the case when the condition is not met
})}
</Select>
<Text className="mt-1">
Select Customer Name
</Text>
<Select defaultValue="all-customers">
<SelectItem
key="all-customers"
value="all-customers"
onClick={() => {
setSelectedCustomer(null);
}}
>
All Customers
</SelectItem>
{
allEndUsers?.map((user: any, index: number) => {
return (
<SelectItem
key={index}
value={user}
onClick={() => {
setSelectedCustomer(user);
}}
>
{user}
</SelectItem>
);
})
}
</Select>
</div>
): (
<div>
<Select defaultValue="all-keys">
<SelectItem
key="all-keys"
value="all-keys"
onClick={() => {
setSelectedAPIKey(null);
}}
>
All Keys
</SelectItem>
{keys?.map((key: any, index: number) => {
if (
key &&
key["key_alias"] !== null &&
key["key_alias"].length > 0
) {
return (
<SelectItem
key={index}
value={String(index)}
// @ts-ignore
disabled={true}
onClick={() => {
setSelectedAPIKey(key);
}}
>
✨ {key["key_alias"]} (Enterpise only Feature)
</SelectItem>
);
}
return null; // Add this line to handle the case when the condition is not met
})}
</Select>
<Text className="mt-1">
Select Customer Name
</Text>
<Select defaultValue="all-customers">
<SelectItem
key="all-customers"
value="all-customers"
onClick={() => {
setSelectedCustomer(null);
}}
>
All Customers
</SelectItem>
{
allEndUsers?.map((user: any, index: number) => {
return (
<SelectItem
key={index}
value={user}
// @ts-ignore
disabled={true}
onClick={() => {
setSelectedCustomer(user);
}}
>
✨ {user} (Enterpise only Feature)
</SelectItem>
);
})
}
</Select>
</div>
)
}
</div>
@@ -1793,12 +1935,13 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</Card>
</TabPanel>
<TabPanel>
<Grid numItems={3} className="mt-2 mb-2">
<Grid numItems={4} className="mt-2 mb-2">
<Col>
<Text>Select Time Range</Text>
<DateRangePicker
enableSelect={true}
value={dateValue}
className="mr-2"
onValueChange={(value) => {
setDateValue(value);
updateModelMetrics(
@@ -1809,7 +1952,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
}}
/>
</Col>
<Col>
<Col className="ml-2">
<Text>Select Model Group</Text>
<Select
defaultValue={
@@ -1838,8 +1981,10 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</Col>
<Col>
<Popover
trigger="click" content={FilterByContent}
overlayStyle={{
width: "20vw"
}}
>
<Button
icon={FilterIcon}
ui/litellm-dashboard/src/components/networking.tsx
+40 -5
View File
@@ -728,6 +728,7 @@ export const modelMetricsCall = async (
startTime: String | undefined,
endTime: String | undefined,
apiKey: String | null,
customer: String | null
) => {
/**
* Get all models on proxy
@@ -735,7 +736,7 @@ export const modelMetricsCall = async (
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`;
}
// message.info("Requesting model data");
const response = await fetch(url, {
@@ -807,7 +808,8 @@ export const modelMetricsSlowResponsesCall = async (
modelGroup: String | null,
startTime: String | undefined,
endTime: String | undefined,
apiKey: String | null
apiKey: String | null,
customer: String | null
) => {
/**
* Get all models on proxy
@@ -817,7 +819,7 @@ export const modelMetricsSlowResponsesCall = async (
? `${proxyBaseUrl}/model/metrics/slow_responses`
: `/model/metrics/slow_responses`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`;
}
// message.info("Requesting model data");
@@ -851,7 +853,8 @@ export const modelExceptionsCall = async (
modelGroup: String | null,
startTime: String | undefined,
endTime: String | undefined,
apiKey: String | null
apiKey: String | null,
customer: String | null
) => {
/**
* Get all models on proxy
@@ -862,7 +865,7 @@ export const modelExceptionsCall = async (
: `/model/metrics/exceptions`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`;
}
const response = await fetch(url, {
method: "GET",
@@ -1053,6 +1056,38 @@ export const allTagNamesCall = async (
};
export const allEndUsersCall = async (
accessToken: String,
) => {
try {
let url = proxyBaseUrl
? `${proxyBaseUrl}/global/all_end_users`
: `/global/all_end_users`;
console.log("in global/all_end_users call", url);
const response = await fetch(`${url}`, {
method: "GET",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.text();
throw new Error("Network response was not ok");
}
const data = await response.json();
console.log(data);
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const userSpendLogsCall = async (
accessToken: String,
token: String,