From 92cdac2e87525f9c3a9bfca57f31e80fa75df837 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 30 Jul 2025 14:42:30 -0700 Subject: [PATCH] [Feat] v2 updates - tracking DAU, WAU, MAU for coding tool usage + show Daily Usage per User (#13147) * Add user agent analytics endpoints and UI for tracking client metrics Co-authored-by: ishaan * fix user agent analytics * fix getting DAU * fixes for user agent * showing top user agents * on this page remove Success Rate by User Agent * fix linting * add agent activity * cleanup interface * fix ruff * round cost * fix charts * fixes - show DAU, MAU, WAU * move to a diff file * fix * fixes for user agent analytics * fix user_agent_analytics_endpoints * fix mypy linting * fix linting --------- Co-authored-by: Cursor Agent Co-authored-by: ishaan --- .../user_agent_analytics_endpoints.py | 200 +++++++++- .../src/components/networking.tsx | 57 +++ .../src/components/per_user_usage.tsx | 258 +++++++++++++ .../src/components/user_agent_activity.tsx | 365 +++++++++--------- 4 files changed, 688 insertions(+), 192 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/per_user_usage.tsx diff --git a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py index 354a5392b8..f24997d96e 100644 --- a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py +++ b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py @@ -14,7 +14,7 @@ tracked by the system. """ from datetime import datetime, timedelta -from typing import Dict, List, Optional, Set +from typing import Dict, List, Optional, Set, cast from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel @@ -59,6 +59,27 @@ class UserAgentAnalyticsResponse(BaseModel): total_pages: int +class PerUserMetrics(BaseModel): + """Metrics for individual user""" + user_id: str + user_email: Optional[str] = None + user_agent: Optional[str] = None + successful_requests: int = 0 + failed_requests: int = 0 + total_requests: int = 0 + total_tokens: int = 0 + spend: float = 0.0 + + +class PerUserAnalyticsResponse(BaseModel): + """Response for per-user analytics""" + results: List[PerUserMetrics] + total_count: int + page: int + page_size: int + total_pages: int + + async def _get_unique_users_for_tags( prisma_client, tags: List[str], @@ -279,7 +300,7 @@ async def get_user_agent_analytics( daily_data_by_tag_and_date[tag][date_str] = daily_data # Calculate DAU/WAU/MAU for each date and tag combination - unique_dates = set() + unique_dates: set[str] = set() for tag_data in daily_data_by_tag_and_date.values(): unique_dates.update(tag_data.keys()) @@ -455,7 +476,7 @@ async def get_user_agent_summary( } for ua, metrics in user_agent_totals.items() ], - key=lambda x: x["requests"], + key=lambda x: cast(int, x["requests"]), reverse=True, )[:10] # Top 10 @@ -473,4 +494,177 @@ async def get_user_agent_summary( raise HTTPException( status_code=500, detail=f"Failed to fetch user agent summary: {str(e)}", + ) + + +@router.get( + "/tag/user-agent/per-user-analytics", + response_model=PerUserAnalyticsResponse, + tags=["tag management", "user agent analytics"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_per_user_analytics( + start_date: Optional[str] = Query( + default=None, + description="Start date in YYYY-MM-DD format", + ), + end_date: Optional[str] = Query( + default=None, + description="End date in YYYY-MM-DD format", + ), + page: int = Query(default=1, description="Page number for pagination", ge=1), + page_size: int = Query( + default=50, description="Items per page", ge=1, le=1000 + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get per-user analytics including successful requests, tokens, and spend by individual users. + + This endpoint provides usage metrics broken down by individual users based on their + user-agent activity during the specified time period. + + Args: + start_date: Start date for the analytics period (YYYY-MM-DD) + end_date: End date for the analytics period (YYYY-MM-DD) + page: Page number for pagination + page_size: Number of items per page + + Returns: + PerUserAnalyticsResponse: Analytics data broken down by individual users + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if start_date is None or end_date is None: + raise HTTPException( + status_code=400, + detail={"error": "Please provide start_date and end_date"}, + ) + + try: + # Get all user-agent tags from the database + user_agent_tags_records = await prisma_client.db.litellm_dailytagspend.find_many( + where={ + "tag": {"startswith": "User-Agent: "}, + "date": {"gte": start_date, "lte": end_date}, + }, + distinct=["tag"], + ) + + user_agent_tags = [record.tag for record in user_agent_tags_records] + + if not user_agent_tags: + return PerUserAnalyticsResponse( + results=[], + total_count=0, + page=page, + page_size=page_size, + total_pages=0, + ) + + # Get all records for user-agent tags in the date range + tag_records = await prisma_client.db.litellm_dailytagspend.find_many( + where={ + "tag": {"in": user_agent_tags}, + "date": {"gte": start_date, "lte": end_date} + } + ) + + # Get unique api_keys + api_keys = set(record.api_key for record in tag_records if record.api_key) + + if not api_keys: + return PerUserAnalyticsResponse( + results=[], + total_count=0, + page=page, + page_size=page_size, + total_pages=0, + ) + + # Lookup user_id for each api_key + api_key_records = await prisma_client.db.litellm_verificationtoken.find_many( + where={"token": {"in": list(api_keys)}} + ) + + # Create mapping from api_key to user_id + api_key_to_user_id = { + record.token: record.user_id + for record in api_key_records + if record.user_id + } + + # Get user emails for the user_ids + user_ids = list(set(api_key_to_user_id.values())) + user_records = await prisma_client.db.litellm_usertable.find_many( + where={"user_id": {"in": user_ids}} + ) + + # Create mapping from user_id to user_email + user_id_to_email = { + record.user_id: record.user_email + for record in user_records + } + + # Aggregate metrics by user + user_metrics: Dict[str, PerUserMetrics] = {} + + for record in tag_records: + if record.api_key in api_key_to_user_id: + user_id = api_key_to_user_id[record.api_key] + user_agent = _extract_user_agent_from_tag(record.tag) + + if user_id not in user_metrics: + user_metrics[user_id] = PerUserMetrics( + user_id=user_id, + user_email=user_id_to_email.get(user_id), + user_agent=user_agent + ) + else: + # If user agent is different, keep the first one or prioritize certain ones + if user_agent and not user_metrics[user_id].user_agent: + user_metrics[user_id].user_agent = user_agent + + # Aggregate metrics + user_metrics[user_id].successful_requests += record.successful_requests or 0 + user_metrics[user_id].failed_requests += record.failed_requests or 0 + user_metrics[user_id].total_requests += record.api_requests or 0 + # Calculate total_tokens from prompt_tokens + completion_tokens + prompt_tokens = record.prompt_tokens or 0 + completion_tokens = record.completion_tokens or 0 + user_metrics[user_id].total_tokens += int(prompt_tokens + completion_tokens) + user_metrics[user_id].spend += record.spend or 0.0 + + # Convert to list and sort by successful requests (descending) + results = sorted( + list(user_metrics.values()), + key=lambda x: x.successful_requests, + reverse=True + ) + + # Apply pagination + total_count = len(results) + total_pages = (total_count + page_size - 1) // page_size + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + paginated_results = results[start_idx:end_idx] + + return PerUserAnalyticsResponse( + results=paginated_results, + total_count=total_count, + page=page, + page_size=page_size, + total_pages=total_pages, + ) + + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to fetch per-user analytics: {str(e)}", ) \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 6fa1e42654..537d7d15c7 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6551,3 +6551,60 @@ export const userAgentSummaryCall = async ( } }; +export const perUserAnalyticsCall = async ( + accessToken: string, + startTime: Date, + endTime: Date, + page: number = 1, + pageSize: number = 50 +) => { + /** + * Get per-user analytics data + */ + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/tag/user-agent/per-user-analytics` + : `/tag/user-agent/per-user-analytics`; + + const queryParams = new URLSearchParams(); + + // Format dates as YYYY-MM-DD for the API + const formatDate = (date: Date) => { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; + }; + + queryParams.append("start_date", formatDate(startTime)); + queryParams.append("end_date", formatDate(endTime)); + queryParams.append("page", page.toString()); + queryParams.append("page_size", pageSize.toString()); + + const queryString = queryParams.toString(); + if (queryString) { + url += `?${queryString}`; + } + + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to fetch per-user analytics:", error); + throw error; + } +}; + diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx new file mode 100644 index 0000000000..1d019e2f59 --- /dev/null +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -0,0 +1,258 @@ +import React, { useState, useEffect } from "react"; +import { + Card, + Title, + Subtitle, + Table, + TableHead, + TableRow, + TableHeaderCell, + TableBody, + TableCell, + BarChart, + Text, + Button, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, +} from "@tremor/react"; +import { perUserAnalyticsCall } from "./networking"; +import { DateRangePickerValue } from "@tremor/react"; + +interface PerUserMetrics { + user_id: string; + user_email: string | null; + user_agent: string | null; + successful_requests: number; + failed_requests: number; + total_requests: number; + total_tokens: number; + spend: number; +} + +interface PerUserAnalyticsResponse { + results: PerUserMetrics[]; + total_count: number; + page: number; + page_size: number; + total_pages: number; +} + +interface PerUserUsageProps { + accessToken: string | null; + dateValue: DateRangePickerValue; + formatAbbreviatedNumber: (value: number, decimalPlaces?: number) => string; +} + +const PerUserUsage: React.FC = ({ + accessToken, + dateValue, + formatAbbreviatedNumber, +}) => { + const [perUserData, setPerUserData] = useState({ + results: [], + total_count: 0, + page: 1, + page_size: 50, + total_pages: 0, + }); + + const [loading, setLoading] = useState(false); + const [currentPage, setCurrentPage] = useState(1); + + const fetchPerUserData = async () => { + if (!accessToken || !dateValue.from || !dateValue.to) return; + + setLoading(true); + try { + const response = await perUserAnalyticsCall( + accessToken, + dateValue.from, + dateValue.to, + currentPage, + 50 + ); + setPerUserData(response); + } catch (error) { + console.error("Failed to fetch per-user data:", error); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchPerUserData(); + }, [accessToken, dateValue, currentPage]); + + const handleNextPage = () => { + if (currentPage < perUserData.total_pages) { + setCurrentPage(currentPage + 1); + } + }; + + const handlePrevPage = () => { + if (currentPage > 1) { + setCurrentPage(currentPage - 1); + } + }; + + return ( +
+ Per User Usage + Individual developer usage metrics + + + + User Details + Usage Distribution + + + + {/* Tab 1: Existing User Details Table */} + + + + + User ID + User Email + User Agent + Success Generations + Total Tokens + Failed Requests + Total Cost + + + + {perUserData.results.slice(0, 10).map((item: PerUserMetrics, index: number) => ( + + + {item.user_id} + + + {item.user_email || "N/A"} + + + {item.user_agent || "Unknown"} + + + {formatAbbreviatedNumber(item.successful_requests)} + + + {formatAbbreviatedNumber(item.total_tokens)} + + + {formatAbbreviatedNumber(item.failed_requests)} + + + ${formatAbbreviatedNumber(item.spend, 4)} + + + ))} + +
+ + {perUserData.results.length > 10 && ( +
+ + Showing 10 of {perUserData.total_count} results + +
+ + +
+
+ )} + + + {/* Tab 2: Usage Distribution Histogram */} + +
+ User Usage Distribution + Number of users by successful request frequency +
+ + { + // Categorize users by successful request count and user agent + const categories = { + "1-9 requests": { range: [1, 9], agents: {} as Record }, + "10-99 requests": { range: [10, 99], agents: {} as Record }, + "100-999 requests": { range: [100, 999], agents: {} as Record }, + "1K-9.9K requests": { range: [1000, 9999], agents: {} as Record }, + "10K-99.9K requests": { range: [10000, 99999], agents: {} as Record }, + "100K+ requests": { range: [100000, Infinity], agents: {} as Record }, + }; + + // Count users in each category by user agent + perUserData.results.forEach((item: PerUserMetrics) => { + const successCount = item.successful_requests; + const userAgent = item.user_agent || "Unknown"; + + Object.entries(categories).forEach(([categoryName, category]) => { + if (successCount >= category.range[0] && successCount <= category.range[1]) { + if (!category.agents[userAgent]) { + category.agents[userAgent] = 0; + } + category.agents[userAgent]++; + } + }); + }); + + // Get all unique user agents + const allUserAgents = new Set(); + Object.values(categories).forEach(category => { + Object.keys(category.agents).forEach(agent => allUserAgents.add(agent)); + }); + const userAgentsList = Array.from(allUserAgents).sort(); + + // Convert to chart data format for stacked bar chart + return Object.entries(categories).map(([categoryName, category]) => { + const dataPoint: Record = { category: categoryName }; + + // Add count for each user agent + userAgentsList.forEach(agent => { + dataPoint[agent] = category.agents[agent] || 0; + }); + + return dataPoint; + }); + })()} + index="category" + categories={(() => { + // Get all unique user agents for the chart categories + const allUserAgents = new Set(); + perUserData.results.forEach((item: PerUserMetrics) => { + allUserAgents.add(item.user_agent || "Unknown"); + }); + return Array.from(allUserAgents).sort(); + })()} + colors={["blue", "green", "orange", "red", "purple", "yellow", "pink", "indigo"]} + valueFormatter={(value: number) => `${value} users`} + yAxisWidth={80} + showLegend={true} + stack={true} + /> + + + +
+ ); +}; + +export default PerUserUsage; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/user_agent_activity.tsx b/ui/litellm-dashboard/src/components/user_agent_activity.tsx index 8ba2a4df32..ec2d1dfb15 100644 --- a/ui/litellm-dashboard/src/components/user_agent_activity.tsx +++ b/ui/litellm-dashboard/src/components/user_agent_activity.tsx @@ -18,10 +18,16 @@ import { Select, SelectItem, Button, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, } from "@tremor/react"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { userAgentAnalyticsCall, userAgentSummaryCall } from "./networking"; import UsageDatePicker from "./shared/usage_date_picker"; +import PerUserUsage from "./per_user_usage"; import { DateRangePickerValue } from "@tremor/react"; interface UserAgentMetrics { @@ -184,20 +190,71 @@ const UserAgentActivity: React.FC = ({ total_requests: ua.requests, })); - // Format numbers with K, M abbreviations - const formatAbbreviatedNumber = (value: number): string => { - if (value >= 100000000) { - return (value / 1000000).toFixed(0) + 'M'; - } else if (value >= 10000000) { - return (value / 1000000).toFixed(0) + 'M'; - } else if (value >= 1000000) { - return (value / 1000000).toFixed(0) + 'M'; - } else if (value >= 10000) { - return (value / 1000).toFixed(0) + 'K'; - } else if (value >= 1000) { - return (value / 1000).toFixed(0) + 'K'; + // Get unique user agents for chart + const uniqueUserAgents = Array.from( + new Set(analyticsData.results.map(item => item.user_agent || "Unknown")) + ).slice(0, 3); // Top 3 user agents + + // Prepare daily chart data (DAU) + const dailyChartData = analyticsData.results.reduce((acc, item) => { + const existingDate = acc.find(d => d.date === item.date); + if (existingDate) { + existingDate[item.user_agent || "Unknown"] = item.metrics.dau; } else { - return value.toString(); + const newDateEntry: any = { + date: item.date, + [item.user_agent || "Unknown"]: item.metrics.dau + }; + acc.push(newDateEntry); + } + return acc; + }, [] as any[]); + + // Prepare weekly chart data (WAU) + const weeklyChartData = analyticsData.results.reduce((acc, item) => { + const existingDate = acc.find(d => d.week === item.date); + if (existingDate) { + existingDate[item.user_agent || "Unknown"] = item.metrics.wau; + } else { + const newDateEntry: any = { + week: `Week ${acc.length + 1}`, + [item.user_agent || "Unknown"]: item.metrics.wau + }; + acc.push(newDateEntry); + } + return acc; + }, [] as any[]); + + // Prepare monthly chart data (MAU) + const monthlyChartData = analyticsData.results.reduce((acc, item) => { + const existingDate = acc.find(d => d.month === item.date); + if (existingDate) { + existingDate[item.user_agent || "Unknown"] = item.metrics.mau; + } else { + const monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul"]; + const newDateEntry: any = { + month: monthNames[acc.length % 7] || `Month ${acc.length + 1}`, + [item.user_agent || "Unknown"]: item.metrics.mau + }; + acc.push(newDateEntry); + } + return acc; + }, [] as any[]); + + // Format numbers with K, M abbreviations + const formatAbbreviatedNumber = (value: number, decimalPlaces: number = 0): string => { + if (value >= 100000000) { + return (value / 1000000).toFixed(decimalPlaces) + 'M'; + } else if (value >= 10000000) { + return (value / 1000000).toFixed(decimalPlaces) + 'M'; + } else if (value >= 1000000) { + return (value / 1000000).toFixed(decimalPlaces) + 'M'; + } else if (value >= 10000) { + return (value / 1000).toFixed(decimalPlaces) + 'K'; + } else if (value >= 1000) { + return (value / 1000).toFixed(decimalPlaces) + 'K'; + } else { + return value.toFixed(decimalPlaces); } }; @@ -238,196 +295,126 @@ const UserAgentActivity: React.FC = ({ {ua.user_agent} - - {formatAbbreviatedNumber(ua.successful_requests)} - - Successful Requests - - {formatAbbreviatedNumber(ua.tokens)} tokens - +
+
+ Success Requests + {formatAbbreviatedNumber(ua.successful_requests)} +
+
+ Total Tokens + {formatAbbreviatedNumber(ua.tokens)} +
+
+ Total Cost + ${formatAbbreviatedNumber(ua.spend, 4)} +
+
))} {/* Fill remaining slots if less than 4 agents */} {Array.from({ length: Math.max(0, 4 - summaryData.top_user_agents.length) }).map((_, index) => ( No Data - - - Successful Requests - - - tokens - +
+
+ Success Requests + - +
+
+ Total Tokens + - +
+
+ Total Cost + - +
+
))} - {/* Charts */} - - - Success Generations by User Agent - formatAbbreviatedNumber(value)} - yAxisWidth={60} - showLegend={true} - /> - - - - {/* Top User Agents Table */} + {/* Main TabGroup for DAU/WAU/MAU vs Per User Usage */} - Top User Agents - - - - User Agent - Requests - Success Rate - Tokens - Spend - - - - {summaryData.top_user_agents.map((ua, index) => ( - - {ua.user_agent} - - {formatNumberWithCommas(ua.requests)} - - - 95 - ? "text-green-600" - : (ua.successful_requests / ua.requests) * 100 > 90 - ? "text-yellow-600" - : "text-red-600" - } - > - {((ua.successful_requests / ua.requests) * 100).toFixed(1)}% - - - - {formatNumberWithCommas(ua.tokens)} - - - ${formatNumberWithCommas(ua.spend, 2)} - - - ))} - -
- + + + DAU/WAU/MAU + Per User Usage + + + + {/* DAU/WAU/MAU Tab Panel */} + +
+ DAU, WAU & MAU per Agent + Active users across different time periods +
- {/* Daily Activity Table */} - -
- Daily User Agent Activity -
- - - Page {currentPage} of {analyticsData.total_pages} - - -
-
- - - - - Date - User Agent - DAU - WAU - MAU - Requests - Success Rate - Tokens - Spend - - - - {analyticsData.results.map((item, index) => ( - - {item.date} - - {item.user_agent || "Unknown"} - - - {formatNumberWithCommas(item.metrics.dau)} - - - {formatNumberWithCommas(item.metrics.wau)} - - - {formatNumberWithCommas(item.metrics.mau)} - - -
-
{formatNumberWithCommas(item.metrics.total_requests)}
-
- {formatNumberWithCommas(item.metrics.successful_requests)} success + + + DAU + WAU + MAU + + + + +
+ Daily Active Users - Last 7 Days
-
- - - 95 - ? "text-green-600" - : (item.metrics.successful_requests / item.metrics.total_requests) * 100 > 90 - ? "text-yellow-600" - : "text-red-600" - } - > - {item.metrics.total_requests > 0 - ? ((item.metrics.successful_requests / item.metrics.total_requests) * 100).toFixed(1) - : "0"}% - - - -
-
{formatNumberWithCommas(item.metrics.total_tokens)}
-
- {formatNumberWithCommas(item.metrics.completed_tokens)} completed + formatAbbreviatedNumber(value)} + yAxisWidth={60} + showLegend={true} + /> + + + +
+ Weekly Active Users - Last 4 Weeks
-
- - - ${formatNumberWithCommas(item.metrics.spend, 4)} - - - ))} - -
- - {analyticsData.results.length === 0 && !loading && ( -
- No user agent activity data found for the selected period. -
- )} - - {loading && ( -
- Loading... -
- )} + formatAbbreviatedNumber(value)} + yAxisWidth={60} + showLegend={true} + /> + + + +
+ Monthly Active Users - Last 7 Months +
+ formatAbbreviatedNumber(value)} + yAxisWidth={60} + showLegend={true} + /> + + + + + + {/* Per User Usage Tab Panel */} + + + + + );