diff --git a/src/web-server/models-dev/registry-cache.ts b/src/web-server/models-dev/registry-cache.ts index 9a2415d4..9b2a0802 100644 --- a/src/web-server/models-dev/registry-cache.ts +++ b/src/web-server/models-dev/registry-cache.ts @@ -9,6 +9,8 @@ const CACHE_FILE_NAME = 'models-dev-registry-cache.json'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; const LIVE_FETCH_TIMEOUT_MS = 3000; +let pendingBackgroundRefresh: Promise | null = null; + export interface RegistryCacheReadOptions { allowStale?: boolean; now?: number; @@ -131,3 +133,17 @@ export async function refreshModelsDevRegistry( clearTimeout(timeoutId); } } + +export function startModelsDevRegistryRefresh( + options: RegistryRefreshOptions = {} +): Promise { + if (!pendingBackgroundRefresh) { + pendingBackgroundRefresh = refreshModelsDevRegistry(options) + .catch(() => null) + .finally(() => { + pendingBackgroundRefresh = null; + }); + } + + return pendingBackgroundRefresh; +} diff --git a/src/web-server/usage/aggregator.ts b/src/web-server/usage/aggregator.ts index 5496ceb8..44df5b21 100644 --- a/src/web-server/usage/aggregator.ts +++ b/src/web-server/usage/aggregator.ts @@ -34,7 +34,8 @@ import { } from './cliproxy-usage-syncer'; import { scanCodexNativeUsageEntries } from './codex-native-usage-collector'; import { scanDroidNativeUsageEntries } from './droid-native-usage-collector'; -import { refreshModelsDevRegistry } from '../models-dev/registry-cache'; +import { startModelsDevRegistryRefresh } from '../models-dev/registry-cache'; +import { getModelsUsed, getProviderModelKey } from './model-identity'; // ============================================================================ // Multi-Instance Support - Aggregate usage from CCS profiles @@ -112,10 +113,6 @@ function getHourlyRequestCount(hour: HourlyUsage): number { return hour.requestCount ?? hour.modelBreakdowns.length; } -function getModelBreakdownKey(breakdown: { modelName: string; provider?: string }): string { - return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; -} - /** * Merge daily usage data from multiple sources * Combines entries with same date by aggregating tokens @@ -133,14 +130,11 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.cacheCreationTokens += day.cacheCreationTokens; existing.cacheReadTokens += day.cacheReadTokens; existing.totalCost += day.totalCost; - // Merge unique models - const modelSet = new Set([...existing.modelsUsed, ...day.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns by aggregating same modelName for (const breakdown of day.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => getModelBreakdownKey(b) === breakdownKey + (b) => getProviderModelKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -152,12 +146,14 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { // Clone to avoid mutating original + const modelBreakdowns = day.modelBreakdowns.map((b) => ({ ...b })); dateMap.set(day.date, { ...day, - modelsUsed: [...day.modelsUsed], - modelBreakdowns: day.modelBreakdowns.map((b) => ({ ...b })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, }); } } @@ -181,12 +177,10 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { existing.cacheCreationTokens += month.cacheCreationTokens; existing.cacheReadTokens += month.cacheReadTokens; existing.totalCost += month.totalCost; - const modelSet = new Set([...existing.modelsUsed, ...month.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); for (const breakdown of month.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (item) => getModelBreakdownKey(item) === breakdownKey + (item) => getProviderModelKey(item) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -198,11 +192,13 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { + const modelBreakdowns = month.modelBreakdowns.map((breakdown) => ({ ...breakdown })); monthMap.set(month.month, { ...month, - modelsUsed: [...month.modelsUsed], - modelBreakdowns: month.modelBreakdowns.map((breakdown) => ({ ...breakdown })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, }); } } @@ -228,13 +224,11 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.cacheReadTokens += hour.cacheReadTokens; existing.totalCost += hour.totalCost; existing.requestCount = getHourlyRequestCount(existing) + getHourlyRequestCount(hour); - const modelSet = new Set([...existing.modelsUsed, ...hour.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns for (const breakdown of hour.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => getModelBreakdownKey(b) === breakdownKey + (b) => getProviderModelKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -246,11 +240,13 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { + const modelBreakdowns = hour.modelBreakdowns.map((b) => ({ ...b })); hourMap.set(hour.hour, { ...hour, - modelsUsed: [...hour.modelsUsed], - modelBreakdowns: hour.modelBreakdowns.map((b) => ({ ...b })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, requestCount: getHourlyRequestCount(hour), }); } @@ -356,9 +352,9 @@ async function refreshFromSource(): Promise<{ monthly: MonthlyUsage[]; session: SessionUsage[]; }> { - // Refresh model metadata before cost derivation. This is best-effort and - // falls back to stale cache/static pricing when models.dev is unavailable. - await refreshModelsDevRegistry(); + // Keep model metadata warming off the analytics request path. Current + // refreshes use cached/static pricing; the background result helps future runs. + void startModelsDevRegistryRefresh(); // Try to sync CLIProxy snapshot before reading it. // Non-fatal: syncer handles unavailability and stale fallback. diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index f33b8e97..8ee7ac21 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -8,6 +8,7 @@ import type { CliproxyUsageApiResponse, CliproxyRequestDetail } from '../../cliproxy/stats-fetcher'; import { calculateCost } from '../model-pricing'; import type { ModelBreakdown, DailyUsage, HourlyUsage, MonthlyUsage } from './types'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // INTERNAL HELPERS @@ -257,10 +258,6 @@ function sumField(breakdowns: ModelBreakdown[], field: keyof ModelBreakdown): nu return breakdowns.reduce((acc, b) => acc + (b[field] as number), 0); } -function getModelsUsed(breakdowns: ModelBreakdown[]): string[] { - return [...new Set(breakdowns.map((breakdown) => breakdown.modelName))]; -} - // ============================================================================ // TRANSFORMS // ============================================================================ diff --git a/src/web-server/usage/data-aggregator.ts b/src/web-server/usage/data-aggregator.ts index 7640051a..e140ae56 100644 --- a/src/web-server/usage/data-aggregator.ts +++ b/src/web-server/usage/data-aggregator.ts @@ -14,6 +14,7 @@ import { type MonthlyUsage, type SessionUsage, } from './types'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // HELPER FUNCTIONS @@ -91,10 +92,6 @@ function createModelAccumulator(entry: RawUsageEntry): ModelAccumulator { }; } -function getModelsUsed(modelMap: Map): string[] { - return [...new Set(Array.from(modelMap.values()).map((acc) => acc.modelName))]; -} - // ============================================================================ // DAILY AGGREGATION // ============================================================================ @@ -173,7 +170,7 @@ export function aggregateDailyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, }); } @@ -262,7 +259,7 @@ export function aggregateHourlyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, requestCount: hourEntries.length, }); @@ -351,7 +348,7 @@ export function aggregateMonthlyUsage( cacheCreationTokens: totalCacheCreation, cacheReadTokens: totalCacheRead, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, }); } @@ -469,7 +466,7 @@ export function aggregateSessionUsage( totalCost, lastActivity, versions: Array.from(versions), - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, source, target, diff --git a/src/web-server/usage/handlers.ts b/src/web-server/usage/handlers.ts index d119075b..b88aa37b 100644 --- a/src/web-server/usage/handlers.ts +++ b/src/web-server/usage/handlers.ts @@ -17,6 +17,7 @@ import { getLastFetchTimestamp, refreshUsageCache, } from './aggregator'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // Types @@ -716,7 +717,6 @@ export async function handleMonthly( cacheCreationTokens: number; cacheReadTokens: number; totalCost: number; - modelsUsed: Set; modelBreakdowns: Map< string, { @@ -741,7 +741,6 @@ export async function handleMonthly( cacheCreationTokens: 0, cacheReadTokens: 0, totalCost: 0, - modelsUsed: new Set(), modelBreakdowns: new Map(), }; @@ -750,9 +749,6 @@ export async function handleMonthly( existing.cacheCreationTokens += day.cacheCreationTokens; existing.cacheReadTokens += day.cacheReadTokens; existing.totalCost += day.totalCost; - for (const model of day.modelsUsed) { - existing.modelsUsed.add(model); - } for (const breakdown of day.modelBreakdowns) { const breakdownKey = getBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.get(breakdownKey) ?? { @@ -783,8 +779,8 @@ export async function handleMonthly( cacheCreationTokens: month.cacheCreationTokens, cacheReadTokens: month.cacheReadTokens, totalCost: month.totalCost, - modelsUsed: Array.from(month.modelsUsed), modelBreakdowns: Array.from(month.modelBreakdowns.values()), + modelsUsed: getModelsUsed(Array.from(month.modelBreakdowns.values())), })) .sort((a, b) => a.month.localeCompare(b.month)); } else { diff --git a/src/web-server/usage/model-identity.ts b/src/web-server/usage/model-identity.ts new file mode 100644 index 00000000..bd2e5ecd --- /dev/null +++ b/src/web-server/usage/model-identity.ts @@ -0,0 +1,37 @@ +export interface ProviderModelIdentity { + modelName: string; + provider?: string; +} + +function getProviderKey(provider: string | undefined): string { + return provider?.trim().toLowerCase() ?? ''; +} + +function getModelUsageLabel(item: ProviderModelIdentity, ambiguousModelNames: Set): string { + const provider = getProviderKey(item.provider); + if (provider && ambiguousModelNames.has(item.modelName)) { + return `${provider}/${item.modelName}`; + } + return item.modelName; +} + +export function getProviderModelKey(item: ProviderModelIdentity): string { + return `${getProviderKey(item.provider)}\u0000${item.modelName}`; +} + +export function getModelsUsed(items: ProviderModelIdentity[]): string[] { + const providersByModel = new Map>(); + for (const item of items) { + const providers = providersByModel.get(item.modelName) ?? new Set(); + providers.add(getProviderKey(item.provider)); + providersByModel.set(item.modelName, providers); + } + + const ambiguousModelNames = new Set( + Array.from(providersByModel.entries()) + .filter(([, providers]) => providers.size > 1) + .map(([modelName]) => modelName) + ); + + return [...new Set(items.map((item) => getModelUsageLabel(item, ambiguousModelNames)))]; +} diff --git a/tests/integration/proxy/daemon-lifecycle.test.ts b/tests/integration/proxy/daemon-lifecycle.test.ts index 0d694a3c..403d438f 100644 --- a/tests/integration/proxy/daemon-lifecycle.test.ts +++ b/tests/integration/proxy/daemon-lifecycle.test.ts @@ -395,13 +395,13 @@ describe('openai proxy daemon lifecycle', () => { }); it('keeps the existing proxy running if replacement startup fails', async () => { - const firstPort = await getPort(); const busyServer = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => new Response('busy'), }); const occupiedPort = busyServer.port; + const firstPort = await getPort({ exclude: [occupiedPort] }); try { const settingsPath = path.join(tempDir, 'rollback.settings.json'); diff --git a/tests/unit/data-aggregator.test.ts b/tests/unit/data-aggregator.test.ts index 4984cc15..3f18f185 100644 --- a/tests/unit/data-aggregator.test.ts +++ b/tests/unit/data-aggregator.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from 'bun:test'; import { aggregateDailyUsage, + aggregateHourlyUsage, aggregateMonthlyUsage, aggregateSessionUsage, } from '../../src/web-server/data-aggregator'; @@ -107,6 +108,36 @@ describe('aggregateDailyUsage', () => { const result = aggregateDailyUsage(entries, 'test-source'); expect(result[0].source).toBe('test-source'); }); + + test('keeps provider identity when same model appears under multiple providers', () => { + const entries: RawUsageEntry[] = [ + createEntry({ model: 'gpt-5.5', target: 'openai' }), + createEntry({ model: 'gpt-5.5', target: 'github-copilot' }), + ]; + + const daily = aggregateDailyUsage(entries); + const hourly = aggregateHourlyUsage(entries); + const monthly = aggregateMonthlyUsage(entries); + const session = aggregateSessionUsage(entries); + + expect(daily[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(hourly[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(monthly[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(session[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(daily[0].modelBreakdowns.map((item) => item.provider)).toEqual([ + 'openai', + 'github-copilot', + ]); + }); + + test('preserves model-only modelsUsed entries when provider is unambiguous', () => { + const result = aggregateDailyUsage([ + createEntry({ model: 'gpt-5.5', target: 'openai' }), + createEntry({ model: 'gpt-5.5', target: 'openai' }), + ]); + + expect(result[0].modelsUsed).toEqual(['gpt-5.5']); + }); }); // ============================================================================ diff --git a/tests/unit/models-dev-registry-cache.test.ts b/tests/unit/models-dev-registry-cache.test.ts index cf640114..20b3dfb5 100644 --- a/tests/unit/models-dev-registry-cache.test.ts +++ b/tests/unit/models-dev-registry-cache.test.ts @@ -8,6 +8,7 @@ import { getCachedModelsDevRegistry, refreshModelsDevRegistry, setCachedModelsDevRegistry, + startModelsDevRegistryRefresh, } from '../../src/web-server/models-dev/registry-cache'; describe('models.dev registry cache', () => { @@ -90,4 +91,48 @@ describe('models.dev registry cache', () => { fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json'); expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); }); + + it('starts and coalesces background refreshes without requiring callers to await', async () => { + let fetchCalls = 0; + let resolveResponse: (response: Response) => void = () => undefined; + const responsePromise = new Promise((resolve) => { + resolveResponse = resolve; + }); + const fetchImpl: typeof fetch = async () => { + fetchCalls += 1; + return responsePromise; + }; + + const first = startModelsDevRegistryRefresh({ + force: true, + fetchImpl, + now: () => 456, + }); + const second = startModelsDevRegistryRefresh({ + force: true, + fetchImpl, + now: () => 789, + }); + + expect(first).toBe(second); + expect(fetchCalls).toBe(1); + expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); + + resolveResponse( + new Response( + JSON.stringify({ + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + }), + { status: 200 } + ) + ); + + await first; + expect(getCachedModelsDevRegistry({ allowStale: true })?.openai.id).toBe('openai'); + }); });