Merge pull request #19953 from BerriAI/litellm_key_alias_spend_usage_report

[Feature] UI - Usage Export: Breakdown by Teams and Keys
This commit is contained in:
yuneng-jiang
2026-01-28 20:29:22 -08:00
committed by GitHub
4 changed files with 661 additions and 2 deletions
@@ -17,11 +17,19 @@ const ExportTypeSelector: React.FC<ExportTypeSelectorProps> = ({ value, onChange
<label className="flex items-start p-3 border border-gray-200 rounded-lg hover:bg-gray-50 cursor-pointer transition-colors">
<Radio value="daily" className="mt-0.5" />
<div className="ml-3 flex-1">
<div className="font-medium text-sm">Day-by-day breakdown</div>
<div className="font-medium text-sm">Day-by-day breakdown by {entityType}</div>
<div className="text-xs text-gray-500 mt-0.5">Daily metrics for each {entityType}</div>
</div>
</label>
<label className="flex items-start p-3 border border-gray-200 rounded-lg hover:bg-gray-50 cursor-pointer transition-colors">
<Radio value="daily_with_keys" className="mt-0.5" />
<div className="ml-3 flex-1">
<div className="font-medium text-sm">Day-by-day breakdown by {entityType} and key</div>
<div className="text-xs text-gray-500 mt-0.5">Daily metrics for each {entityType}, split by API key</div>
</div>
</label>
<label className="flex items-start p-3 border border-gray-200 rounded-lg hover:bg-gray-50 cursor-pointer transition-colors">
<Radio value="daily_with_models" className="mt-0.5" />
<div className="ml-3 flex-1">
@@ -2,7 +2,7 @@ import type { DateRangePickerValue } from "@tremor/react";
import type { Team } from "@/components/key_team_helpers/key_list";
export type ExportFormat = "csv" | "json";
export type ExportScope = "daily" | "daily_with_models";
export type ExportScope = "daily" | "daily_with_keys" | "daily_with_models";
export type EntityType = "tag" | "team" | "organization" | "customer" | "agent";
export interface EntitySpendData {
@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { EntitySpendData, ExportScope } from "./types";
import {
generateDailyData,
generateDailyWithKeysData,
generateDailyWithModelsData,
generateExportData,
generateMetadata,
@@ -462,6 +463,514 @@ describe("EntityUsageExport utils", () => {
});
});
describe("generateDailyWithKeysData", () => {
const mockSpendDataWithKeys: EntitySpendData = {
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 5.0,
api_requests: 50,
successful_requests: 48,
failed_requests: 2,
total_tokens: 500,
prompt_tokens: 300,
completion_tokens: 200,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
key2: {
metrics: {
spend: 5.5,
api_requests: 50,
successful_requests: 47,
failed_requests: 3,
total_tokens: 500,
prompt_tokens: 300,
completion_tokens: 200,
},
metadata: {
team_id: "team-1",
key_alias: "alias-2",
},
},
},
},
entity2: {
metrics: {
spend: 20.3,
api_requests: 200,
successful_requests: 190,
failed_requests: 10,
total_tokens: 2000,
prompt_tokens: 1200,
completion_tokens: 800,
},
api_key_breakdown: {
key3: {
metrics: {
spend: 20.3,
api_requests: 200,
successful_requests: 190,
failed_requests: 10,
total_tokens: 2000,
prompt_tokens: 1200,
completion_tokens: 800,
},
metadata: {
team_id: "team-2",
key_alias: "alias-3",
},
},
},
},
},
},
},
{
date: "2025-01-02",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 15.2,
api_requests: 150,
successful_requests: 145,
failed_requests: 5,
total_tokens: 1500,
prompt_tokens: 900,
completion_tokens: 600,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 15.2,
api_requests: 150,
successful_requests: 145,
failed_requests: 5,
total_tokens: 1500,
prompt_tokens: 900,
completion_tokens: 600,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
},
},
},
},
},
],
metadata: {
total_spend: 46.0,
total_api_requests: 450,
total_successful_requests: 430,
total_failed_requests: 20,
total_tokens: 4500,
},
};
it("should generate daily breakdown with key data and correct structure", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team", mockTeamAliasMap);
expect(result.length).toBeGreaterThan(0);
expect(result[0]).toHaveProperty("Date");
expect(result[0]).toHaveProperty("Team");
expect(result[0]).toHaveProperty("Team ID");
expect(result[0]).toHaveProperty("Key Alias");
expect(result[0]).toHaveProperty("Key ID");
expect(result[0]).toHaveProperty("Spend ($)");
expect(result[0]).toHaveProperty("Requests");
expect(result[0]).toHaveProperty("Successful Requests");
expect(result[0]).toHaveProperty("Failed Requests");
expect(result[0]).toHaveProperty("Total Tokens");
expect(result[0]).toHaveProperty("Prompt Tokens");
expect(result[0]).toHaveProperty("Completion Tokens");
});
it("should sort data by date ascending", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team");
const dates = result.map((r) => new Date(r.Date).getTime());
for (let i = 0; i < dates.length - 1; i++) {
expect(dates[i]).toBeLessThanOrEqual(dates[i + 1]);
}
});
it("should aggregate metrics for duplicate date-team-key combinations", () => {
const spendDataWithDuplicates: EntitySpendData = {
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 5.0,
api_requests: 50,
successful_requests: 48,
failed_requests: 2,
total_tokens: 500,
prompt_tokens: 300,
completion_tokens: 200,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
},
},
},
},
},
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 5.0,
api_requests: 50,
successful_requests: 47,
failed_requests: 3,
total_tokens: 500,
prompt_tokens: 300,
completion_tokens: 200,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
},
},
},
},
},
],
metadata: {
total_spend: 21.0,
total_api_requests: 200,
total_successful_requests: 190,
total_failed_requests: 10,
total_tokens: 2000,
},
};
const result = generateDailyWithKeysData(spendDataWithDuplicates, "Team");
const key1Entries = result.filter((r) => r["Key ID"] === "key1");
expect(key1Entries).toHaveLength(1);
expect(key1Entries[0].Requests).toBe(100);
expect(key1Entries[0]["Successful Requests"]).toBe(95);
expect(key1Entries[0]["Failed Requests"]).toBe(5);
expect(key1Entries[0]["Total Tokens"]).toBe(1000);
});
it("should use team alias when available", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team", mockTeamAliasMap);
const team1Entry = result.find((r) => r["Team ID"] === "team-1");
expect(team1Entry?.["Team"]).toBe("Team One");
});
it("should use dash when team alias is not available", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team");
const entryWithoutTeamAlias = result.find((r) => r["Team ID"] === "team-1" && !mockTeamAliasMap[r["Team ID"]]);
if (entryWithoutTeamAlias) {
expect(entryWithoutTeamAlias["Team"]).toBe("-");
}
});
it("should use key alias when available", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team");
const key1Entry = result.find((r) => r["Key ID"] === "key1");
expect(key1Entry?.["Key Alias"]).toBe("alias-1");
});
it("should use dash when key alias is not available", () => {
const spendDataWithoutKeyAlias: EntitySpendData = {
...mockSpendDataWithKeys,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
metadata: {
team_id: "team-1",
},
},
},
},
},
},
},
],
metadata: mockSpendDataWithKeys.metadata,
};
const result = generateDailyWithKeysData(spendDataWithoutKeyAlias, "Team");
const key1Entry = result.find((r) => r["Key ID"] === "key1");
expect(key1Entry?.["Key Alias"]).toBe("-");
});
it("should use entity id when team id is not available in metadata", () => {
const spendDataWithoutTeamId: EntitySpendData = {
...mockSpendDataWithKeys,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
metadata: {},
},
},
},
},
},
},
],
metadata: mockSpendDataWithKeys.metadata,
};
const result = generateDailyWithKeysData(spendDataWithoutTeamId, "Team");
const entry = result.find((r) => r["Key ID"] === "key1");
expect(entry?.["Team ID"]).toBe("entity1");
});
it("should use dash when team id is not available", () => {
const spendDataWithoutTeamId: EntitySpendData = {
...mockSpendDataWithKeys,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
metadata: {
team_id: null,
},
},
},
},
},
},
},
],
metadata: mockSpendDataWithKeys.metadata,
};
const result = generateDailyWithKeysData(spendDataWithoutTeamId, "Team");
const entry = result.find((r) => r["Key ID"] === "key1");
expect(entry?.["Team ID"]).toBe("entity1");
});
it("should format spend values correctly", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team");
expect(result[0]["Spend ($)"]).toBeDefined();
});
it("should handle missing optional token fields", () => {
const spendDataWithMissingTokens: EntitySpendData = {
...mockSpendDataWithKeys,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
},
},
},
},
},
],
metadata: mockSpendDataWithKeys.metadata,
};
const result = generateDailyWithKeysData(spendDataWithMissingTokens, "Team");
const key1Entry = result.find((r) => r["Key ID"] === "key1");
expect(key1Entry?.["Prompt Tokens"]).toBe(0);
expect(key1Entry?.["Completion Tokens"]).toBe(0);
});
it("should handle empty api_key_breakdown", () => {
const spendDataWithEmptyBreakdown: EntitySpendData = {
...mockSpendDataWithKeys,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {},
},
},
},
},
],
metadata: mockSpendDataWithKeys.metadata,
};
const result = generateDailyWithKeysData(spendDataWithEmptyBreakdown, "Team");
expect(result).toHaveLength(0);
});
it("should handle multiple keys for same team on same date", () => {
const result = generateDailyWithKeysData(mockSpendDataWithKeys, "Team");
const team1Entries = result.filter((r) => r["Team ID"] === "team-1" && r.Date === "2025-01-01");
expect(team1Entries.length).toBeGreaterThanOrEqual(2);
const keyIds = team1Entries.map((r) => r["Key ID"]);
expect(keyIds).toContain("key1");
expect(keyIds).toContain("key2");
});
});
describe("generateDailyWithModelsData", () => {
const mockSpendDataWithModels: EntitySpendData = {
results: [
@@ -698,6 +1207,57 @@ describe("EntityUsageExport utils", () => {
expect(result[0]).not.toHaveProperty("Model");
});
it("should return daily with keys data when scope is daily_with_keys", () => {
const mockDataWithKeys: EntitySpendData = {
...mockSpendData,
results: [
{
date: "2025-01-01",
breakdown: {
entities: {
entity1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
api_key_breakdown: {
key1: {
metrics: {
spend: 10.5,
api_requests: 100,
successful_requests: 95,
failed_requests: 5,
total_tokens: 1000,
prompt_tokens: 600,
completion_tokens: 400,
},
metadata: {
team_id: "team-1",
key_alias: "alias-1",
},
},
},
},
},
},
},
],
metadata: mockSpendData.metadata,
};
const result = generateExportData(mockDataWithKeys, "daily_with_keys", "Team", mockTeamAliasMap);
expect(result.length).toBeGreaterThan(0);
expect(result[0]).toHaveProperty("Key Alias");
expect(result[0]).toHaveProperty("Key ID");
expect(result[0]).not.toHaveProperty("Model");
});
it("should return daily with models data when scope is daily_with_models", () => {
const mockDataWithModels: EntitySpendData = {
...mockSpendData,
@@ -103,6 +103,95 @@ export const generateDailyData = (
return dailyBreakdown.sort((a, b) => new Date(a.Date).getTime() - new Date(b.Date).getTime());
};
export const generateDailyWithKeysData = (
spendData: EntitySpendData,
entityLabel: string,
teamAliasMap: Record<string, string> = {},
): any[] => {
// Aggregate by unique (Date, Team ID, Key ID) combination to prevent duplicates
const aggregatedData: {
[key: string]: {
Date: string;
teamId: string;
teamAlias: string | null;
keyId: string;
keyAlias: string | null;
metrics: {
spend: number;
api_requests: number;
successful_requests: number;
failed_requests: number;
total_tokens: number;
prompt_tokens: number;
completion_tokens: number;
};
};
} = {};
spendData.results.forEach((day) => {
Object.entries(day.breakdown.entities || {}).forEach(([entity, data]: [string, any]) => {
const apiKeyBreakdown = data.api_key_breakdown || {};
// Iterate through each API key in the breakdown
Object.entries(apiKeyBreakdown).forEach(([keyId, keyData]: [string, any]) => {
const keyAlias = keyData?.metadata?.key_alias || null;
const teamId = keyData?.metadata?.team_id || entity;
const teamAlias = teamId ? teamAliasMap[teamId] || null : null;
// Create unique key for aggregation: Date_TeamID_KeyID
const uniqueKey = `${day.date}_${teamId}_${keyId}`;
if (!aggregatedData[uniqueKey]) {
// First time seeing this (Date, Team ID, Key ID) combination
aggregatedData[uniqueKey] = {
Date: day.date,
teamId,
teamAlias,
keyId,
keyAlias,
metrics: {
spend: keyData.metrics?.spend || 0,
api_requests: keyData.metrics?.api_requests || 0,
successful_requests: keyData.metrics?.successful_requests || 0,
failed_requests: keyData.metrics?.failed_requests || 0,
total_tokens: keyData.metrics?.total_tokens || 0,
prompt_tokens: keyData.metrics?.prompt_tokens || 0,
completion_tokens: keyData.metrics?.completion_tokens || 0,
},
};
} else {
// Aggregate metrics for existing entry
aggregatedData[uniqueKey].metrics.spend += keyData.metrics?.spend || 0;
aggregatedData[uniqueKey].metrics.api_requests += keyData.metrics?.api_requests || 0;
aggregatedData[uniqueKey].metrics.successful_requests += keyData.metrics?.successful_requests || 0;
aggregatedData[uniqueKey].metrics.failed_requests += keyData.metrics?.failed_requests || 0;
aggregatedData[uniqueKey].metrics.total_tokens += keyData.metrics?.total_tokens || 0;
aggregatedData[uniqueKey].metrics.prompt_tokens += keyData.metrics?.prompt_tokens || 0;
aggregatedData[uniqueKey].metrics.completion_tokens += keyData.metrics?.completion_tokens || 0;
}
});
});
});
// Convert aggregated data to array format
const dailyKeyBreakdown = Object.values(aggregatedData).map((item) => ({
Date: item.Date,
[entityLabel]: item.teamAlias || "-",
[`${entityLabel} ID`]: item.teamId || "-",
"Key Alias": item.keyAlias || "-",
"Key ID": item.keyId,
"Spend ($)": formatNumberWithCommas(item.metrics.spend, 4),
Requests: item.metrics.api_requests,
"Successful Requests": item.metrics.successful_requests,
"Failed Requests": item.metrics.failed_requests,
"Total Tokens": item.metrics.total_tokens,
"Prompt Tokens": item.metrics.prompt_tokens,
"Completion Tokens": item.metrics.completion_tokens,
}));
return dailyKeyBreakdown.sort((a, b) => new Date(a.Date).getTime() - new Date(b.Date).getTime());
};
export const generateDailyWithModelsData = (
spendData: EntitySpendData,
entityLabel: string,
@@ -174,6 +263,8 @@ export const generateExportData = (
switch (exportScope) {
case "daily":
return generateDailyData(spendData, entityLabel, teamAliasMap);
case "daily_with_keys":
return generateDailyWithKeysData(spendData, entityLabel, teamAliasMap);
case "daily_with_models":
return generateDailyWithModelsData(spendData, entityLabel, teamAliasMap);
default: