mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-23 22:27:10 +00:00
Merge pull request #23622 from BerriAI/litellm_usage_page_auto_pagination
[Feature] UI - Usage: Auto-paginate daily spend data
This commit is contained in:
yuneng-jiang
+127
-105
@@ -22,7 +22,9 @@ import {
|
||||
Text,
|
||||
Title,
|
||||
} from "@tremor/react";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { ExportOutlined, LoadingOutlined } from "@ant-design/icons";
|
||||
import { Alert, Button } from "antd";
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { ActivityMetrics, processActivityData } from "../../../activity_metrics";
|
||||
import { UsageExportHeader } from "../../../EntityUsageExport";
|
||||
import type { EntityType } from "../../../EntityUsageExport/types";
|
||||
@@ -35,6 +37,7 @@ import {
|
||||
userDailyActivityCall,
|
||||
} from "../../../networking";
|
||||
import { getProviderLogoAndName } from "../../../provider_info_helpers";
|
||||
import { usePaginatedDailyActivity } from "../../hooks/usePaginatedDailyActivity";
|
||||
import { BreakdownMetrics, DailyData, EntityMetricWithMetadata, KeyMetricWithMetadata, TagUsage } from "../../types";
|
||||
import { valueFormatterSpend } from "../../utils/value_formatters";
|
||||
import EndpointUsage from "../EndpointUsage/EndpointUsage";
|
||||
@@ -87,119 +90,64 @@ interface EntityUsageProps {
|
||||
dateValue: DateRangePickerValue;
|
||||
}
|
||||
|
||||
const ENTITY_FETCH_FNS: Record<EntityType, (...args: any[]) => Promise<any>> = {
|
||||
tag: tagDailyActivityCall,
|
||||
team: teamDailyActivityCall,
|
||||
organization: organizationDailyActivityCall,
|
||||
customer: customerDailyActivityCall,
|
||||
agent: agentDailyActivityCall,
|
||||
user: userDailyActivityCall,
|
||||
};
|
||||
|
||||
const EntityUsage: React.FC<EntityUsageProps> = ({ accessToken, entityType, entityId, entityList, dateValue }) => {
|
||||
const [spendData, setSpendData] = useState<EntitySpendData>({
|
||||
results: [],
|
||||
metadata: {
|
||||
total_spend: 0,
|
||||
total_api_requests: 0,
|
||||
total_successful_requests: 0,
|
||||
total_failed_requests: 0,
|
||||
total_tokens: 0,
|
||||
},
|
||||
});
|
||||
const { teams } = useTeams();
|
||||
|
||||
const [agentSpendData, setAgentSpendData] = useState<EntitySpendData>({
|
||||
results: [],
|
||||
metadata: {
|
||||
total_spend: 0,
|
||||
total_api_requests: 0,
|
||||
total_successful_requests: 0,
|
||||
total_failed_requests: 0,
|
||||
total_tokens: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const modelMetrics = processActivityData(spendData, "models", teams || []);
|
||||
const keyMetrics = processActivityData(spendData, "api_keys", teams || []);
|
||||
const agentMetrics = entityType === "team" ? processActivityData(agentSpendData, "entities", teams || []) : {};
|
||||
const [selectedTags, setSelectedTags] = useState<string[]>([]);
|
||||
const [topKeysLimit, setTopKeysLimit] = useState<number>(5);
|
||||
const [topModelsLimit, setTopModelsLimit] = useState<number>(5);
|
||||
const [topAgentsLimit, setTopAgentsLimit] = useState<number>(5);
|
||||
|
||||
const fetchSpendData = async () => {
|
||||
if (!accessToken || !dateValue.from || !dateValue.to) return;
|
||||
// Create new Date objects to avoid mutating the original dates
|
||||
const startTime = new Date(dateValue.from);
|
||||
const endTime = new Date(dateValue.to);
|
||||
const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]);
|
||||
const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]);
|
||||
|
||||
if (entityType === "tag") {
|
||||
const data = await tagDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else if (entityType === "team") {
|
||||
const data = await teamDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else if (entityType === "organization") {
|
||||
const data = await organizationDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else if (entityType === "customer") {
|
||||
const data = await customerDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else if (entityType === "agent") {
|
||||
const data = await agentDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else if (entityType === "user") {
|
||||
const data = await userDailyActivityCall(
|
||||
accessToken,
|
||||
startTime,
|
||||
endTime,
|
||||
1,
|
||||
selectedTags.length > 0 ? selectedTags[0] : null,
|
||||
);
|
||||
setSpendData(data);
|
||||
} else {
|
||||
throw new Error("Invalid entity type");
|
||||
}
|
||||
};
|
||||
const entityFilterArg = useMemo(() => {
|
||||
if (entityType === "user") return selectedTags.length > 0 ? selectedTags[0] : null;
|
||||
return selectedTags.length > 0 ? selectedTags : null;
|
||||
}, [entityType, selectedTags]);
|
||||
|
||||
const fetchAgentSpendData = async () => {
|
||||
if (!accessToken || !dateValue.from || !dateValue.to || entityType !== "team") return;
|
||||
const startTime = new Date(dateValue.from);
|
||||
const endTime = new Date(dateValue.to);
|
||||
try {
|
||||
const data = await agentDailyActivityCall(accessToken, startTime, endTime, 1, null);
|
||||
setAgentSpendData(data);
|
||||
} catch (e) {
|
||||
console.error("Failed to fetch agent activity data:", e);
|
||||
}
|
||||
};
|
||||
const fetchFn = ENTITY_FETCH_FNS[entityType];
|
||||
const enabled = !!accessToken && !!startTime && !!endTime;
|
||||
|
||||
useEffect(() => {
|
||||
fetchSpendData();
|
||||
fetchAgentSpendData();
|
||||
}, [accessToken, dateValue, entityId, selectedTags]);
|
||||
const {
|
||||
data: spendDataRaw,
|
||||
isFetchingMore,
|
||||
progress,
|
||||
cancelled,
|
||||
cancel,
|
||||
} = usePaginatedDailyActivity({
|
||||
fetchFn,
|
||||
args: [accessToken, startTime, endTime, entityFilterArg],
|
||||
enabled,
|
||||
});
|
||||
|
||||
const spendData = spendDataRaw as unknown as EntitySpendData;
|
||||
|
||||
const {
|
||||
data: agentSpendDataRaw,
|
||||
isFetchingMore: agentIsFetchingMore,
|
||||
progress: agentProgress,
|
||||
cancelled: agentCancelled,
|
||||
cancel: agentCancel,
|
||||
} = usePaginatedDailyActivity({
|
||||
fetchFn: agentDailyActivityCall,
|
||||
args: [accessToken, startTime, endTime, null],
|
||||
enabled: enabled && entityType === "team",
|
||||
});
|
||||
|
||||
const agentSpendData = agentSpendDataRaw as unknown as EntitySpendData;
|
||||
|
||||
const modelMetrics = processActivityData(spendData, "models", teams || []);
|
||||
const keyMetrics = processActivityData(spendData, "api_keys", teams || []);
|
||||
const agentMetrics = entityType === "team" ? processActivityData(agentSpendData, "entities", teams || []) : {};
|
||||
|
||||
const getTopModels = () => {
|
||||
const modelSpend: { [key: string]: any } = {};
|
||||
@@ -448,6 +396,78 @@ const EntityUsage: React.FC<EntityUsageProps> = ({ accessToken, entityType, enti
|
||||
|
||||
return (
|
||||
<div style={{ width: "100%" }} className="relative">
|
||||
{isFetchingMore && (
|
||||
<Alert
|
||||
banner
|
||||
type="warning"
|
||||
className="mb-2"
|
||||
message={
|
||||
<div className="flex items-center justify-between">
|
||||
<span>
|
||||
<LoadingOutlined spin className="mr-2" />
|
||||
Currently fetching spend data: fetched {progress.currentPage} / {progress.totalPages} pages. Charts will
|
||||
update periodically as data loads. Moving off of this page will stop and reset this. To continue using
|
||||
the UI in the meantime,{" "}
|
||||
<a href={window.location.href} target="_blank" rel="noopener noreferrer">
|
||||
open a new tab <ExportOutlined />
|
||||
</a>
|
||||
.
|
||||
</span>
|
||||
<Button type="primary" danger onClick={cancel}>
|
||||
Stop
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{cancelled && (
|
||||
<Alert
|
||||
banner
|
||||
type="info"
|
||||
className="mb-2"
|
||||
message={
|
||||
<span>
|
||||
Showing partial data ({progress.currentPage}/{progress.totalPages} pages loaded)
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{agentIsFetchingMore && entityType === "team" && (
|
||||
<Alert
|
||||
banner
|
||||
type="warning"
|
||||
className="mb-2"
|
||||
message={
|
||||
<div className="flex items-center justify-between">
|
||||
<span>
|
||||
<LoadingOutlined spin className="mr-2" />
|
||||
Currently fetching agent data: fetched {agentProgress.currentPage} / {agentProgress.totalPages} pages.
|
||||
Charts will update periodically as data loads. Moving off of this page will stop and reset this. To
|
||||
continue using the UI in the meantime,{" "}
|
||||
<a href={window.location.href} target="_blank" rel="noopener noreferrer">
|
||||
open a new tab <ExportOutlined />
|
||||
</a>
|
||||
.
|
||||
</span>
|
||||
<Button type="primary" danger onClick={agentCancel}>
|
||||
Stop
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{agentCancelled && entityType === "team" && (
|
||||
<Alert
|
||||
banner
|
||||
type="info"
|
||||
className="mb-2"
|
||||
message={
|
||||
<span>
|
||||
Showing partial agent data ({agentProgress.currentPage}/{agentProgress.totalPages} pages loaded)
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
<UsageExportHeader
|
||||
dateValue={dateValue}
|
||||
entityType={entityType}
|
||||
@@ -772,7 +792,9 @@ const EntityUsage: React.FC<EntityUsageProps> = ({ accessToken, entityType, enti
|
||||
<TabPanel>
|
||||
<ActivityMetrics modelMetrics={agentMetrics} />
|
||||
</TabPanel>
|
||||
) : <></>}
|
||||
) : (
|
||||
<></>
|
||||
)}
|
||||
<TabPanel>
|
||||
<ActivityMetrics modelMetrics={keyMetrics} hidePromptCachingMetrics={entityType === "agent"} />
|
||||
</TabPanel>
|
||||
|
||||
@@ -251,6 +251,7 @@ vi.mock("@ant-design/icons", async () => {
|
||||
UserOutlined: Icon,
|
||||
DownOutlined: Icon,
|
||||
RightOutlined: Icon,
|
||||
ExportOutlined: Icon,
|
||||
LoadingOutlined,
|
||||
};
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,246 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { DailyData } from "../types";
|
||||
|
||||
export interface PaginationProgress {
|
||||
currentPage: number;
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
/** Delay between sequential page fetches (ms) to avoid overloading the backend. */
|
||||
const PAGE_FETCH_DELAY_MS = 300;
|
||||
|
||||
/** Number of pages to accumulate before flushing to React state (reduces re-renders). */
|
||||
const RENDER_BATCH_SIZE = 3;
|
||||
|
||||
/** The metadata fields returned by the daily activity API that should be summed across pages. */
|
||||
const SUMMABLE_METADATA_KEYS = [
|
||||
"total_spend",
|
||||
"total_prompt_tokens",
|
||||
"total_completion_tokens",
|
||||
"total_tokens",
|
||||
"total_api_requests",
|
||||
"total_successful_requests",
|
||||
"total_failed_requests",
|
||||
"total_cache_read_input_tokens",
|
||||
"total_cache_creation_input_tokens",
|
||||
] as const;
|
||||
|
||||
interface DailyActivityResponse {
|
||||
results: DailyData[];
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
type FetchPageFn = (...args: any[]) => Promise<DailyActivityResponse>;
|
||||
|
||||
interface UsePaginatedDailyActivityParams {
|
||||
/** The API call function (e.g., userDailyActivityCall). */
|
||||
fetchFn: FetchPageFn;
|
||||
/** Arguments to pass to fetchFn: [accessToken, startTime, endTime, ...extraArgs]. Page is injected by the hook at index 3. */
|
||||
args: any[];
|
||||
/** Whether the hook should fetch. Set to false to disable. */
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
interface UsePaginatedDailyActivityReturn {
|
||||
data: DailyActivityResponse;
|
||||
loading: boolean;
|
||||
isFetchingMore: boolean;
|
||||
progress: PaginationProgress;
|
||||
cancelled: boolean;
|
||||
cancel: () => void;
|
||||
}
|
||||
|
||||
const EMPTY_DATA: DailyActivityResponse = {
|
||||
results: [],
|
||||
metadata: {
|
||||
total_spend: 0,
|
||||
total_prompt_tokens: 0,
|
||||
total_completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
total_api_requests: 0,
|
||||
total_successful_requests: 0,
|
||||
total_failed_requests: 0,
|
||||
total_cache_read_input_tokens: 0,
|
||||
total_cache_creation_input_tokens: 0,
|
||||
total_pages: 1,
|
||||
has_more: false,
|
||||
page: 1,
|
||||
},
|
||||
};
|
||||
|
||||
function sumMetadata(
|
||||
a: Record<string, any>,
|
||||
b: Record<string, any>,
|
||||
): Record<string, any> {
|
||||
const result = { ...a };
|
||||
for (const key of SUMMABLE_METADATA_KEYS) {
|
||||
result[key] = (a[key] || 0) + (b[key] || 0);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook that auto-paginates daily activity endpoints, updating state in batches
|
||||
* so charts render progressively. Cancels on unmount, param changes, or
|
||||
* manual cancel().
|
||||
*
|
||||
* The `args` array should contain every argument the fetchFn expects EXCEPT
|
||||
* the `page` parameter. The hook injects `page` as the 4th argument (index 3),
|
||||
* matching the signature of all daily activity calls:
|
||||
* (accessToken, startTime, endTime, page, ...rest)
|
||||
*/
|
||||
export function usePaginatedDailyActivity({
|
||||
fetchFn,
|
||||
args,
|
||||
enabled,
|
||||
}: UsePaginatedDailyActivityParams): UsePaginatedDailyActivityReturn {
|
||||
const [data, setData] = useState<DailyActivityResponse>(EMPTY_DATA);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [isFetchingMore, setIsFetchingMore] = useState(false);
|
||||
const [progress, setProgress] = useState<PaginationProgress>({
|
||||
currentPage: 0,
|
||||
totalPages: 0,
|
||||
});
|
||||
const [cancelled, setCancelled] = useState(false);
|
||||
|
||||
const fetchIdRef = useRef(0);
|
||||
const cancelledRef = useRef(false);
|
||||
const delayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Keep args in a ref so the effect can always read the latest values
|
||||
// without needing them in the dependency array.
|
||||
const argsRef = useRef(args);
|
||||
argsRef.current = args;
|
||||
|
||||
// Stable serialised key so the effect only re-runs when the arg *values* change.
|
||||
const argsKey = JSON.stringify(args);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
setCancelled(true);
|
||||
setIsFetchingMore(false);
|
||||
if (delayTimerRef.current !== null) {
|
||||
clearTimeout(delayTimerRef.current);
|
||||
delayTimerRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
setData(EMPTY_DATA);
|
||||
setLoading(false);
|
||||
setIsFetchingMore(false);
|
||||
setProgress({ currentPage: 0, totalPages: 0 });
|
||||
setCancelled(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const currentFetchId = ++fetchIdRef.current;
|
||||
cancelledRef.current = false;
|
||||
setCancelled(false);
|
||||
|
||||
const isStale = () =>
|
||||
fetchIdRef.current !== currentFetchId || cancelledRef.current;
|
||||
|
||||
/** Cancellable delay that clears itself on cleanup. */
|
||||
const delay = (ms: number) =>
|
||||
new Promise<void>((resolve) => {
|
||||
delayTimerRef.current = setTimeout(() => {
|
||||
delayTimerRef.current = null;
|
||||
resolve();
|
||||
}, ms);
|
||||
});
|
||||
|
||||
const run = async () => {
|
||||
const currentArgs = argsRef.current;
|
||||
setLoading(true);
|
||||
setIsFetchingMore(false);
|
||||
setProgress({ currentPage: 1, totalPages: 1 });
|
||||
|
||||
try {
|
||||
// Inject page=1 as the 4th argument.
|
||||
const argsWithPage = [...currentArgs.slice(0, 3), 1, ...currentArgs.slice(3)];
|
||||
const firstPage = await fetchFn(...argsWithPage);
|
||||
|
||||
if (isStale()) return;
|
||||
|
||||
setData(firstPage);
|
||||
|
||||
const totalPages = firstPage.metadata?.total_pages || 1;
|
||||
|
||||
setProgress({ currentPage: 1, totalPages });
|
||||
|
||||
if (totalPages <= 1) {
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// More pages — start fetching sequentially.
|
||||
setLoading(false);
|
||||
setIsFetchingMore(true);
|
||||
|
||||
let accumulatedResults = [...firstPage.results];
|
||||
let accumulatedMetadata = { ...firstPage.metadata };
|
||||
|
||||
for (let page = 2; page <= totalPages; page++) {
|
||||
if (isStale()) return;
|
||||
|
||||
// Small delay to avoid overwhelming the backend.
|
||||
await delay(PAGE_FETCH_DELAY_MS);
|
||||
|
||||
if (isStale()) return;
|
||||
|
||||
const argsForPage = [...currentArgs.slice(0, 3), page, ...currentArgs.slice(3)];
|
||||
const pageData = await fetchFn(...argsForPage);
|
||||
|
||||
if (isStale()) return;
|
||||
|
||||
accumulatedResults = [...accumulatedResults, ...pageData.results];
|
||||
accumulatedMetadata = sumMetadata(
|
||||
accumulatedMetadata,
|
||||
pageData.metadata,
|
||||
);
|
||||
accumulatedMetadata.total_pages = totalPages;
|
||||
accumulatedMetadata.has_more = page < totalPages;
|
||||
accumulatedMetadata.page = page;
|
||||
|
||||
// Flush accumulated data and progress to React state every
|
||||
// RENDER_BATCH_SIZE pages (or on the final page) to avoid
|
||||
// expensive per-page re-renders. Progress and data are updated
|
||||
// together so the counter never appears to decrement.
|
||||
const isLastPage = page === totalPages;
|
||||
const isBatchBoundary = (page - 1) % RENDER_BATCH_SIZE === 0;
|
||||
if (isLastPage || isBatchBoundary) {
|
||||
setData({
|
||||
results: accumulatedResults,
|
||||
metadata: accumulatedMetadata,
|
||||
});
|
||||
setProgress({ currentPage: page, totalPages });
|
||||
}
|
||||
}
|
||||
|
||||
setIsFetchingMore(false);
|
||||
} catch (error) {
|
||||
if (!isStale()) {
|
||||
console.error("Error fetching daily activity:", error);
|
||||
setLoading(false);
|
||||
setIsFetchingMore(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
run();
|
||||
|
||||
return () => {
|
||||
fetchIdRef.current++;
|
||||
if (delayTimerRef.current !== null) {
|
||||
clearTimeout(delayTimerRef.current);
|
||||
delayTimerRef.current = null;
|
||||
}
|
||||
};
|
||||
// argsKey is a stable JSON string so the effect only re-fires when arg values change.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [enabled, fetchFn, argsKey]);
|
||||
|
||||
return { data, loading, isFetchingMore, progress, cancelled, cancel };
|
||||
}
|
||||
@@ -77,6 +77,14 @@ vi.mock("@/components/networking", () => {
|
||||
// Called when decoding a valid token
|
||||
setGlobalLitellmHeaderName: vi.fn(),
|
||||
Organization: {},
|
||||
// Daily activity calls used by UsagePage components in the render tree
|
||||
tagDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
teamDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
organizationDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
customerDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
agentDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
userDailyActivityCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
userDailyActivityAggregatedCall: vi.fn().mockResolvedValue({ results: [], metadata: {} }),
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user