From a38c1a75ba73606c68262a16d2e488c30f939da8 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 14:50:26 -0400 Subject: [PATCH 1/7] feat: integrate models.dev pricing metadata --- docs/codebase-summary.md | 10 +- docs/project-roadmap.md | 1 + src/web-server/model-pricing.ts | 48 ++++- src/web-server/models-dev/pricing-resolver.ts | 191 ++++++++++++++++++ src/web-server/models-dev/registry-cache.ts | 133 ++++++++++++ src/web-server/models-dev/types.ts | 42 ++++ src/web-server/usage/aggregator.ts | 18 +- .../usage/cliproxy-usage-transformer.ts | 79 +++++--- src/web-server/usage/data-aggregator.ts | 102 +++++----- src/web-server/usage/handlers.ts | 23 ++- src/web-server/usage/types.ts | 1 + tests/unit/model-pricing.test.ts | 90 ++++++++- tests/unit/models-dev-registry-cache.test.ts | 93 +++++++++ .../cliproxy-usage-transformer.test.ts | 108 +++++++++- 14 files changed, 851 insertions(+), 88 deletions(-) create mode 100644 src/web-server/models-dev/pricing-resolver.ts create mode 100644 src/web-server/models-dev/registry-cache.ts create mode 100644 src/web-server/models-dev/types.ts create mode 100644 tests/unit/models-dev-registry-cache.test.ts diff --git a/docs/codebase-summary.md b/docs/codebase-summary.md index cdff9ac7..5d203769 100644 --- a/docs/codebase-summary.md +++ b/docs/codebase-summary.md @@ -2,7 +2,7 @@ Last Updated: 2026-04-26 -Comprehensive overview of the modularized CCS codebase structure following the Phase 9 modularization effort (Settings, Analytics, Auth Monitor splits + Test Infrastructure), v7.1 Remote CLIProxy feature, v7.2 Kiro + GitHub Copilot (ghcp) OAuth providers, v7.14 Hybrid Quota Management, v7.34 Image Analysis Hook, account-context validation hardening, Official Claude Channels runtime support, native Codex runtime target support, and native Codex/Droid usage collectors. +Comprehensive overview of the modularized CCS codebase structure following the Phase 9 modularization effort (Settings, Analytics, Auth Monitor splits + Test Infrastructure), v7.1 Remote CLIProxy feature, v7.2 Kiro + GitHub Copilot (ghcp) OAuth providers, v7.14 Hybrid Quota Management, v7.34 Image Analysis Hook, account-context validation hardening, Official Claude Channels runtime support, native Codex runtime target support, native Codex/Droid usage collectors, and models.dev-backed model pricing metadata. ## Repository Structure @@ -219,9 +219,13 @@ src/ │ ├── codex-native-usage-collector.ts # Native Codex rollout JSONL collector │ ├── droid-native-usage-collector.ts # Native Droid SQLite collector │ └── data-aggregator.ts + ├── models-dev/ # Cached models.dev metadata/pricing registry integration + │ ├── registry-cache.ts + │ ├── pricing-resolver.ts + │ └── types.ts ├── services/ # Shared services │ └── index.ts - └── model-pricing.ts # Model cost definitions (676 lines) + └── model-pricing.ts # Static pricing fallback + models.dev resolver ``` ### Module Categories @@ -550,7 +554,7 @@ ui/src/ | File | Lines | Status | |------|-------|--------| -| model-pricing.ts | 676 | Data file - acceptable | +| model-pricing.ts | 920 | Static pricing fallback and resolver entrypoint | | glmt-proxy.ts | 675 | Legacy internal compatibility path - acceptable for now | | cliproxy-executor.ts | 666 | Core logic - acceptable | | cliproxy-command.ts | 634 | Could split if needed | diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index f439fae7..68d18131 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -81,6 +81,7 @@ All major modularization work is complete. The codebase evolved from monolithic - **#724**: Codex startup is now free-plan safe. CCS defaults new Codex sessions to a cross-plan model and uses runtime fallback handling for unsupported paid-only models without rewriting the saved dashboard settings. - **#737**: Dashboard model pickers in Cursor, Copilot, and CLIProxy now use a searchable combobox with autofocus and explicit no-results states for large model catalogs. - **#736**: `ccs config` now supports explicit dashboard bind hosts via `--host`, and surfaces remote-access warnings plus reachable URLs when the effective bind is non-loopback. +- **#1121**: Usage analytics pricing now refreshes cached models.dev metadata before cost derivation, keeps CCS static pricing as the offline fallback, and carries provider identity through CLIProxy/native runtime breakdowns so subscription-backed providers do not inherit paid API pricing. ### Maintainability Hardening Kickoff diff --git a/src/web-server/model-pricing.ts b/src/web-server/model-pricing.ts index 2c1e6766..e7c7f0c1 100644 --- a/src/web-server/model-pricing.ts +++ b/src/web-server/model-pricing.ts @@ -7,6 +7,12 @@ * All rates are in USD per MILLION tokens. */ +import { + getKnownModelsDevModels, + resolveModelsDevPricing, + type ModelsDevPricingLookupOptions, +} from './models-dev/pricing-resolver'; + // ============================================================================ // TYPE DEFINITIONS // ============================================================================ @@ -25,6 +31,8 @@ export interface TokenUsage { cacheReadTokens: number; } +export type PricingLookupOptions = ModelsDevPricingLookupOptions; + // ============================================================================ // USER-EDITABLE PRICING TABLE // Update rates below (per million tokens in USD) @@ -827,17 +835,38 @@ function getDirectOrAliasPricing(model: string): ModelPricing | undefined { return undefined; } +function hasProviderContext(model: string, options: PricingLookupOptions): boolean { + return Boolean(options.provider || /^[^/]+\//.test(model.trim())); +} + /** * Get pricing for a model with narrow fuzzy matching fallback. * Unknown future model families should fall back instead of inheriting the * first known family tier that happens to share a prefix. */ -export function getModelPricing(model: string): ModelPricing { +export function getModelPricing(model: string, options: PricingLookupOptions = {}): ModelPricing { + const exactPricing = PRICING_REGISTRY[model]; + if (exactPricing !== undefined) { + return exactPricing; + } + + if (hasProviderContext(model, options)) { + const providerPricing = resolveModelsDevPricing(model, options); + if (providerPricing !== undefined) { + return providerPricing.pricing; + } + } + const directOrAliasPricing = getDirectOrAliasPricing(model); if (directOrAliasPricing !== undefined) { return directOrAliasPricing; } + const modelsDevPricing = resolveModelsDevPricing(model, options); + if (modelsDevPricing !== undefined) { + return modelsDevPricing.pricing; + } + for (const candidate of getLookupCandidates(model)) { // Allow provider/routing wrappers to suffix a canonical model ID. for (const [key, pricing] of Object.entries(NORMALIZED_PRICING_REGISTRY)) { @@ -857,8 +886,12 @@ export function getModelPricing(model: string): ModelPricing { * @param model - Model name for pricing lookup * @returns Cost in USD */ -export function calculateCost(usage: TokenUsage, model: string): number { - const pricing = getModelPricing(model); +export function calculateCost( + usage: TokenUsage, + model: string, + options: PricingLookupOptions = {} +): number { + const pricing = getModelPricing(model, options); const inputCost = (usage.inputTokens / 1_000_000) * pricing.inputPerMillion; const outputCost = (usage.outputTokens / 1_000_000) * pricing.outputPerMillion; @@ -873,12 +906,15 @@ export function calculateCost(usage: TokenUsage, model: string): number { * Get list of all known models for UI display */ export function getKnownModels(): string[] { - return Object.keys(PRICING_REGISTRY); + return [...new Set([...Object.keys(PRICING_REGISTRY), ...getKnownModelsDevModels()])]; } /** * Check if a model has custom pricing (not using fallback) */ -export function hasCustomPricing(model: string): boolean { - return getDirectOrAliasPricing(model) !== undefined; +export function hasCustomPricing(model: string, options: PricingLookupOptions = {}): boolean { + return ( + getDirectOrAliasPricing(model) !== undefined || + resolveModelsDevPricing(model, options) !== undefined + ); } diff --git a/src/web-server/models-dev/pricing-resolver.ts b/src/web-server/models-dev/pricing-resolver.ts new file mode 100644 index 00000000..a45fcd18 --- /dev/null +++ b/src/web-server/models-dev/pricing-resolver.ts @@ -0,0 +1,191 @@ +import { getCachedModelsDevRegistry } from './registry-cache'; +import type { ModelsDevCost, ModelsDevModel, ModelsDevProvider, ModelsDevRegistry } from './types'; + +export interface ModelsDevPricing { + inputPerMillion: number; + outputPerMillion: number; + cacheCreationPerMillion: number; + cacheReadPerMillion: number; +} + +export interface ModelsDevPricingResolution { + provider: string; + model: string; + pricing: ModelsDevPricing; +} + +export interface ModelsDevPricingLookupOptions { + provider?: string; +} + +const PROVIDER_ALIASES: Record = { + agy: 'google', + antigravity: 'google', + claude: 'anthropic', + codex: 'openai', + copilot: 'github-copilot', + gemini: 'google', + ghcp: 'github-copilot', + github: 'github-copilot', + kimi: 'moonshotai', + moonshot: 'moonshotai', + qwen: 'alibaba', +}; + +function normalizeId(value: string): string { + return value.trim().toLowerCase(); +} + +function normalizeProvider(provider: string | undefined): string | undefined { + if (!provider) return undefined; + const normalized = normalizeId(provider); + return PROVIDER_ALIASES[normalized] ?? normalized; +} + +function splitProviderPrefix(model: string): { provider?: string; model: string } { + const trimmed = model.trim(); + const slashIndex = trimmed.indexOf('/'); + if (slashIndex <= 0) return { model: trimmed }; + return { + provider: trimmed.slice(0, slashIndex), + model: trimmed.slice(slashIndex + 1), + }; +} + +function stripClaudeDateSuffix(model: string): string { + if (!model.startsWith('claude-')) return model; + return model.replace(/-\d{8}(?=-thinking(?:$|:))/g, '').replace(/-\d{8}(?=$|:)/g, ''); +} + +function getModelCandidates(model: string): string[] { + const normalized = normalizeId(model); + const baseModel = normalized.split(':')[0]; + const candidates = [normalized]; + + if (baseModel !== normalized) candidates.push(baseModel); + + for (const candidate of [...candidates]) { + const stripped = stripClaudeDateSuffix(candidate); + if (stripped !== candidate && !candidates.includes(stripped)) { + candidates.push(stripped); + } + } + + return candidates; +} + +function findProvider( + registry: ModelsDevRegistry, + provider: string | undefined +): ModelsDevProvider | undefined { + const normalizedProvider = normalizeProvider(provider); + if (!normalizedProvider) return undefined; + return registry[normalizedProvider]; +} + +function findModel(provider: ModelsDevProvider, model: string): ModelsDevModel | undefined { + const models = provider.models; + if (!models) return undefined; + + const normalizedEntries = new Map(); + for (const [key, value] of Object.entries(models)) { + normalizedEntries.set(normalizeId(key), value); + if (typeof value.id === 'string') normalizedEntries.set(normalizeId(value.id), value); + } + + for (const candidate of getModelCandidates(model)) { + const match = normalizedEntries.get(candidate); + if (match) return match; + } + + return undefined; +} + +function toNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function toPricing(cost: ModelsDevCost | null | undefined): ModelsDevPricing | undefined { + const input = toNumber(cost?.input); + const output = toNumber(cost?.output); + if (input === undefined || output === undefined) return undefined; + + return { + inputPerMillion: input, + outputPerMillion: output, + cacheCreationPerMillion: toNumber(cost?.cache_write) ?? 0, + cacheReadPerMillion: toNumber(cost?.cache_read) ?? 0, + }; +} + +function samePricing(left: ModelsDevPricing, right: ModelsDevPricing): boolean { + return ( + left.inputPerMillion === right.inputPerMillion && + left.outputPerMillion === right.outputPerMillion && + left.cacheCreationPerMillion === right.cacheCreationPerMillion && + left.cacheReadPerMillion === right.cacheReadPerMillion + ); +} + +function resolveProviderModel( + registry: ModelsDevRegistry, + providerId: string | undefined, + model: string +): ModelsDevPricingResolution | undefined { + const provider = findProvider(registry, providerId); + if (!provider) return undefined; + + const entry = findModel(provider, model); + const pricing = toPricing(entry?.cost); + if (!entry || !pricing) return undefined; + + return { provider: provider.id, model: entry.id, pricing }; +} + +function resolveUnambiguousModel( + registry: ModelsDevRegistry, + model: string +): ModelsDevPricingResolution | undefined { + const matches: ModelsDevPricingResolution[] = []; + + for (const provider of Object.values(registry)) { + const match = resolveProviderModel(registry, provider.id, model); + if (match) matches.push(match); + } + + if (matches.length === 0) return undefined; + const first = matches[0]; + if (matches.every((match) => samePricing(first.pricing, match.pricing))) { + return first; + } + return undefined; +} + +export function resolveModelsDevPricing( + model: string, + options: ModelsDevPricingLookupOptions = {} +): ModelsDevPricingResolution | undefined { + const registry = getCachedModelsDevRegistry({ allowStale: true }); + if (!registry) return undefined; + + const prefixed = splitProviderPrefix(model); + const provider = options.provider ?? prefixed.provider; + const modelId = prefixed.model; + + return ( + resolveProviderModel(registry, provider, modelId) ?? resolveUnambiguousModel(registry, modelId) + ); +} + +export function getKnownModelsDevModels(): string[] { + const registry = getCachedModelsDevRegistry({ allowStale: true }); + if (!registry) return []; + + const ids = new Set(); + for (const provider of Object.values(registry)) { + for (const model of Object.values(provider.models ?? {})) { + if (typeof model.id === 'string') ids.add(`${provider.id}/${model.id}`); + } + } + return Array.from(ids).sort((a, b) => a.localeCompare(b)); +} diff --git a/src/web-server/models-dev/registry-cache.ts b/src/web-server/models-dev/registry-cache.ts new file mode 100644 index 00000000..9a2415d4 --- /dev/null +++ b/src/web-server/models-dev/registry-cache.ts @@ -0,0 +1,133 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { getCcsDir } from '../../utils/config-manager'; +import type { ModelsDevCacheData, ModelsDevProvider, ModelsDevRegistry } from './types'; + +export const MODELS_DEV_API_URL = 'https://models.dev/api.json'; + +const CACHE_FILE_NAME = 'models-dev-registry-cache.json'; +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const LIVE_FETCH_TIMEOUT_MS = 3000; + +export interface RegistryCacheReadOptions { + allowStale?: boolean; + now?: number; +} + +export interface RegistryRefreshOptions { + force?: boolean; + fetchImpl?: typeof fetch; + now?: () => number; +} + +function getCacheFilePath(): string { + return path.join(getCcsDir(), CACHE_FILE_NAME); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function normalizeRegistryPayload(payload: unknown): ModelsDevRegistry | null { + if (!isPlainObject(payload)) return null; + + const providers: ModelsDevRegistry = {}; + for (const [key, value] of Object.entries(payload)) { + if (!isPlainObject(value)) continue; + const id = typeof value.id === 'string' && value.id.trim() ? value.id.trim() : key; + const models = isPlainObject(value.models) + ? (value.models as NonNullable) + : undefined; + if (!models || Object.keys(models).length === 0) continue; + + providers[id] = { + ...(value as ModelsDevProvider), + id, + models, + }; + } + + return Object.keys(providers).length > 0 ? providers : null; +} + +function normalizeCachePayload(payload: unknown): ModelsDevCacheData | null { + if (!isPlainObject(payload)) return null; + if (payload.version !== 1 || typeof payload.fetchedAt !== 'number') return null; + const providers = normalizeRegistryPayload(payload.providers); + return providers ? { version: 1, fetchedAt: payload.fetchedAt, providers } : null; +} + +export function getCachedModelsDevRegistry( + options: RegistryCacheReadOptions = {} +): ModelsDevRegistry | null { + try { + const filePath = getCacheFilePath(); + if (!fs.existsSync(filePath)) return null; + + const cache = normalizeCachePayload(JSON.parse(fs.readFileSync(filePath, 'utf8'))); + if (!cache) return null; + + const now = options.now ?? Date.now(); + if (!options.allowStale && now - cache.fetchedAt > CACHE_TTL_MS) return null; + return cache.providers; + } catch { + return null; + } +} + +export function setCachedModelsDevRegistry( + providers: ModelsDevRegistry, + fetchedAt = Date.now() +): void { + try { + const filePath = getCacheFilePath(); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const cache: ModelsDevCacheData = { version: 1, fetchedAt, providers }; + fs.writeFileSync(filePath, JSON.stringify(cache)); + } catch { + // Best-effort cache writes must not break analytics. + } +} + +export function clearModelsDevRegistryCache(): boolean { + try { + const filePath = getCacheFilePath(); + if (!fs.existsSync(filePath)) return false; + fs.unlinkSync(filePath); + return true; + } catch { + return false; + } +} + +export async function refreshModelsDevRegistry( + options: RegistryRefreshOptions = {} +): Promise { + const now = options.now ?? (() => Date.now()); + if (!options.force) { + const fresh = getCachedModelsDevRegistry({ allowStale: false, now: now() }); + if (fresh) return fresh; + } + + const fetchImpl = options.fetchImpl ?? fetch; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), LIVE_FETCH_TIMEOUT_MS); + + try { + const response = await fetchImpl(MODELS_DEV_API_URL, { + signal: controller.signal, + headers: { Accept: 'application/json' }, + }); + if (!response.ok) return getCachedModelsDevRegistry({ allowStale: true }); + + const providers = normalizeRegistryPayload(await response.json()); + if (!providers) return getCachedModelsDevRegistry({ allowStale: true }); + + setCachedModelsDevRegistry(providers, now()); + return providers; + } catch { + return getCachedModelsDevRegistry({ allowStale: true }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/src/web-server/models-dev/types.ts b/src/web-server/models-dev/types.ts new file mode 100644 index 00000000..8536c7ee --- /dev/null +++ b/src/web-server/models-dev/types.ts @@ -0,0 +1,42 @@ +export interface ModelsDevCost { + input?: number; + output?: number; + cache_read?: number; + cache_write?: number; + [key: string]: unknown; +} + +export interface ModelsDevModel { + id: string; + name?: string; + cost?: ModelsDevCost | null; + limit?: Record; + modalities?: { + input?: string[]; + output?: string[]; + }; + reasoning?: boolean | null; + tool_call?: boolean | null; + structured_output?: boolean | null; + temperature?: boolean | null; + [key: string]: unknown; +} + +export interface ModelsDevProvider { + id: string; + name?: string; + env?: string[]; + npm?: string; + api?: string | null; + doc?: string; + models?: Record; + [key: string]: unknown; +} + +export type ModelsDevRegistry = Record; + +export interface ModelsDevCacheData { + version: 1; + fetchedAt: number; + providers: ModelsDevRegistry; +} diff --git a/src/web-server/usage/aggregator.ts b/src/web-server/usage/aggregator.ts index 97913930..5496ceb8 100644 --- a/src/web-server/usage/aggregator.ts +++ b/src/web-server/usage/aggregator.ts @@ -34,6 +34,7 @@ import { } from './cliproxy-usage-syncer'; import { scanCodexNativeUsageEntries } from './codex-native-usage-collector'; import { scanDroidNativeUsageEntries } from './droid-native-usage-collector'; +import { refreshModelsDevRegistry } from '../models-dev/registry-cache'; // ============================================================================ // Multi-Instance Support - Aggregate usage from CCS profiles @@ -111,6 +112,10 @@ function getHourlyRequestCount(hour: HourlyUsage): number { return hour.requestCount ?? hour.modelBreakdowns.length; } +function getModelBreakdownKey(breakdown: { modelName: string; provider?: string }): string { + return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; +} + /** * Merge daily usage data from multiple sources * Combines entries with same date by aggregating tokens @@ -133,8 +138,9 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns by aggregating same modelName for (const breakdown of day.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => b.modelName === breakdown.modelName + (b) => getModelBreakdownKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -178,8 +184,9 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { const modelSet = new Set([...existing.modelsUsed, ...month.modelsUsed]); existing.modelsUsed = Array.from(modelSet); for (const breakdown of month.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (item) => item.modelName === breakdown.modelName + (item) => getModelBreakdownKey(item) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -225,8 +232,9 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns for (const breakdown of hour.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => b.modelName === breakdown.modelName + (b) => getModelBreakdownKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -348,6 +356,10 @@ async function refreshFromSource(): Promise<{ monthly: MonthlyUsage[]; session: SessionUsage[]; }> { + // Refresh model metadata before cost derivation. This is best-effort and + // falls back to stale cache/static pricing when models.dev is unavailable. + await refreshModelsDevRegistry(); + // Try to sync CLIProxy snapshot before reading it. // Non-fatal: syncer handles unavailability and stale fallback. await syncCliproxyUsage(); diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index 2ae608e5..f33b8e97 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -16,6 +16,7 @@ import type { ModelBreakdown, DailyUsage, HourlyUsage, MonthlyUsage } from './ty /** Persisted request detail used to rebuild historical CLIProxy analytics buckets */ export interface CliproxyUsageHistoryDetail { model: string; + provider?: string; timestamp: string; source: string; authIndex: string; @@ -36,17 +37,32 @@ interface ModelAccumulator { } /** Build ModelBreakdown from accumulated token counts */ -function buildModelBreakdown(modelName: string, acc: ModelAccumulator): ModelBreakdown { +function buildModelBreakdown( + modelName: string, + provider: string | undefined, + acc: ModelAccumulator +): ModelBreakdown { const { inputTokens, outputTokens, cacheReadTokens, cost } = acc; - return { modelName, inputTokens, outputTokens, cacheCreationTokens: 0, cacheReadTokens, cost }; + return { + modelName, + ...(provider && { provider }), + inputTokens, + outputTokens, + cacheCreationTokens: 0, + cacheReadTokens, + cost, + }; } function createHistoryDetail( + provider: string, model: string, detail: CliproxyRequestDetail ): CliproxyUsageHistoryDetail { + const pricingProvider = provider.trim().toLowerCase(); return { model, + provider: pricingProvider, timestamp: detail.timestamp, source: detail.source, authIndex: String(detail.auth_index), @@ -61,7 +77,8 @@ function createHistoryDetail( cacheCreationTokens: 0, cacheReadTokens: detail.tokens?.cached_tokens ?? 0, }, - model + model, + { provider: pricingProvider } ), failed: detail.failed, }; @@ -92,7 +109,7 @@ export function extractCliproxyUsageHistoryDetails( if (!apis) return []; const results: CliproxyUsageHistoryDetail[] = []; - for (const providerData of Object.values(apis)) { + for (const [provider, providerData] of Object.entries(apis)) { const models = providerData?.models; if (!models) continue; for (const [model, modelData] of Object.entries(models)) { @@ -100,7 +117,7 @@ export function extractCliproxyUsageHistoryDetails( if (!details) continue; for (const detail of details) { if (detail.failed && !hasTrackedUsage(detail)) continue; - results.push(createHistoryDetail(model, detail)); + results.push(createHistoryDetail(provider, model, detail)); } } } @@ -110,6 +127,7 @@ export function extractCliproxyUsageHistoryDetails( function createHistorySignature(detail: CliproxyUsageHistoryDetail): string { return [ detail.model, + detail.provider ?? '', detail.timestamp, detail.source, detail.authIndex, @@ -188,24 +206,35 @@ function aggregateByKey( buildRecord: (key: string, breakdowns: ModelBreakdown[], requestCount: number) => T, sortFn: (a: T, b: T) => number ): T[] { - // bucket: timeKey -> modelName -> accumulator - const buckets = new Map>(); + // bucket: timeKey -> provider/model key -> accumulator + const buckets = new Map< + string, + Map + >(); const requestCounts = new Map(); for (const detail of flat) { const key = keyFn(detail.timestamp); if (!buckets.has(key)) buckets.set(key, new Map()); requestCounts.set(key, (requestCounts.get(key) ?? 0) + detail.requestCount); - const modelMap = buckets.get(key) as Map; - if (!modelMap.has(detail.model)) { - modelMap.set(detail.model, { - inputTokens: 0, - outputTokens: 0, - cacheReadTokens: 0, - cost: 0, + const modelMap = buckets.get(key) as Map< + string, + { modelName: string; provider?: string; acc: ModelAccumulator } + >; + const modelKey = `${detail.provider ?? ''}\u0000${detail.model}`; + if (!modelMap.has(modelKey)) { + modelMap.set(modelKey, { + modelName: detail.model, + provider: detail.provider, + acc: { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cost: 0, + }, }); } - const acc = modelMap.get(detail.model) as ModelAccumulator; + const acc = (modelMap.get(modelKey) as { acc: ModelAccumulator }).acc; acc.inputTokens += detail.inputTokens; acc.outputTokens += detail.outputTokens; acc.cacheReadTokens += detail.cacheReadTokens; @@ -214,8 +243,8 @@ function aggregateByKey( const records: T[] = []; Array.from(buckets.entries()).forEach(([key, modelMap]) => { - const breakdowns = Array.from(modelMap.entries()).map(([name, acc]) => - buildModelBreakdown(name, acc) + const breakdowns = Array.from(modelMap.values()).map((entry) => + buildModelBreakdown(entry.modelName, entry.provider, entry.acc) ); records.push(buildRecord(key, breakdowns, requestCounts.get(key) ?? 0)); }); @@ -228,6 +257,10 @@ function sumField(breakdowns: ModelBreakdown[], field: keyof ModelBreakdown): nu return breakdowns.reduce((acc, b) => acc + (b[field] as number), 0); } +function getModelsUsed(breakdowns: ModelBreakdown[]): string[] { + return [...new Set(breakdowns.map((breakdown) => breakdown.modelName))]; +} + // ============================================================================ // TRANSFORMS // ============================================================================ @@ -249,7 +282,7 @@ export function transformCliproxyToDailyUsage(response: CliproxyUsageApiResponse cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }; }, @@ -278,7 +311,7 @@ export function transformCliproxyToHourlyUsage(response: CliproxyUsageApiRespons cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, requestCount, }; @@ -303,7 +336,7 @@ export function transformCliproxyToMonthlyUsage( cacheCreationTokens: 0, cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), totalCost: sumField(breakdowns, 'cost'), - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }), (a, b) => b.month.localeCompare(a.month) @@ -330,7 +363,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }; }, @@ -354,7 +387,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, requestCount, }; @@ -372,7 +405,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheCreationTokens: 0, cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), totalCost: sumField(breakdowns, 'cost'), - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }), (a, b) => b.month.localeCompare(a.month) diff --git a/src/web-server/usage/data-aggregator.ts b/src/web-server/usage/data-aggregator.ts index ba664576..7640051a 100644 --- a/src/web-server/usage/data-aggregator.ts +++ b/src/web-server/usage/data-aggregator.ts @@ -39,6 +39,7 @@ function extractHour(timestamp: string): string { /** Create model breakdown from accumulated data */ function createModelBreakdown( modelName: string, + provider: string | undefined, inputTokens: number, outputTokens: number, cacheCreationTokens: number, @@ -46,11 +47,13 @@ function createModelBreakdown( ): ModelBreakdown { const cost = calculateCost( { inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens }, - modelName + modelName, + { provider } ); return { modelName, + ...(provider && { provider }), inputTokens, outputTokens, cacheCreationTokens, @@ -61,12 +64,37 @@ function createModelBreakdown( /** Accumulator for per-model token counts */ interface ModelAccumulator { + modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; cacheReadTokens: number; } +function getEntryProvider(entry: RawUsageEntry): string | undefined { + return entry.target?.trim().toLowerCase() || undefined; +} + +function getEntryModelKey(entry: RawUsageEntry): string { + return `${getEntryProvider(entry) ?? ''}\u0000${entry.model}`; +} + +function createModelAccumulator(entry: RawUsageEntry): ModelAccumulator { + return { + modelName: entry.model, + provider: getEntryProvider(entry), + inputTokens: 0, + outputTokens: 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, + }; +} + +function getModelsUsed(modelMap: Map): string[] { + return [...new Set(Array.from(modelMap.values()).map((acc) => acc.modelName))]; +} + // ============================================================================ // DAILY AGGREGATION // ============================================================================ @@ -101,19 +129,14 @@ export function aggregateDailyUsage( let totalCacheRead = 0; for (const entry of dateEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -125,9 +148,10 @@ export function aggregateDailyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -149,7 +173,7 @@ export function aggregateDailyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, }); } @@ -194,19 +218,14 @@ export function aggregateHourlyUsage( let totalCacheRead = 0; for (const entry of hourEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -218,9 +237,10 @@ export function aggregateHourlyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -242,7 +262,7 @@ export function aggregateHourlyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, requestCount: hourEntries.length, }); @@ -288,19 +308,14 @@ export function aggregateMonthlyUsage( let totalCacheRead = 0; for (const entry of monthEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -312,9 +327,10 @@ export function aggregateMonthlyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -335,7 +351,7 @@ export function aggregateMonthlyUsage( cacheCreationTokens: totalCacheCreation, cacheReadTokens: totalCacheRead, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, }); } @@ -388,19 +404,14 @@ export function aggregateSessionUsage( let target: string | undefined; for (const entry of orderedEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -431,9 +442,10 @@ export function aggregateSessionUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -457,7 +469,7 @@ export function aggregateSessionUsage( totalCost, lastActivity, versions: Array.from(versions), - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, source, target, diff --git a/src/web-server/usage/handlers.ts b/src/web-server/usage/handlers.ts index 2f3bb054..d119075b 100644 --- a/src/web-server/usage/handlers.ts +++ b/src/web-server/usage/handlers.ts @@ -141,6 +141,10 @@ function calculateUsageTotalTokens( return input + output + cacheCreation + cacheRead; } +function getBreakdownKey(breakdown: { modelName: string; provider?: string }): string { + return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; +} + function parseDateKey(dateString: string): Date { return new Date( Date.UTC( @@ -204,7 +208,7 @@ export function calculateTokenBreakdownCosts(dailyData: DailyUsage[]): TokenBrea for (const day of dailyData) { for (const breakdown of day.modelBreakdowns) { - const pricing = getModelPricing(breakdown.modelName); + const pricing = getModelPricing(breakdown.modelName, { provider: breakdown.provider }); inputTokens += breakdown.inputTokens; outputTokens += breakdown.outputTokens; cacheCreationTokens += breakdown.cacheCreationTokens; @@ -541,6 +545,7 @@ export async function handleModels( string, { model: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; @@ -551,8 +556,10 @@ export async function handleModels( for (const day of filtered) { for (const breakdown of day.modelBreakdowns) { - const existing = modelMap.get(breakdown.modelName) || { + const modelKey = getBreakdownKey(breakdown); + const existing = modelMap.get(modelKey) || { model: breakdown.modelName, + provider: breakdown.provider, inputTokens: 0, outputTokens: 0, cacheCreationTokens: 0, @@ -564,7 +571,7 @@ export async function handleModels( existing.cacheCreationTokens += breakdown.cacheCreationTokens; existing.cacheReadTokens += breakdown.cacheReadTokens; existing.cost += breakdown.cost; - modelMap.set(breakdown.modelName, existing); + modelMap.set(modelKey, existing); } } @@ -583,7 +590,7 @@ export async function handleModels( const result = models .map((m) => { - const pricing = getModelPricing(m.model); + const pricing = getModelPricing(m.model, { provider: m.provider }); const inputCost = (m.inputTokens / 1_000_000) * pricing.inputPerMillion; const outputCost = (m.outputTokens / 1_000_000) * pricing.outputPerMillion; const cacheCreationCost = @@ -599,6 +606,7 @@ export async function handleModels( return { model: m.model, + provider: m.provider, tokens: totalModelTokens, inputTokens: m.inputTokens, outputTokens: m.outputTokens, @@ -713,6 +721,7 @@ export async function handleMonthly( string, { modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; @@ -745,8 +754,10 @@ export async function handleMonthly( existing.modelsUsed.add(model); } for (const breakdown of day.modelBreakdowns) { - const existingBreakdown = existing.modelBreakdowns.get(breakdown.modelName) ?? { + const breakdownKey = getBreakdownKey(breakdown); + const existingBreakdown = existing.modelBreakdowns.get(breakdownKey) ?? { modelName: breakdown.modelName, + provider: breakdown.provider, inputTokens: 0, outputTokens: 0, cacheCreationTokens: 0, @@ -758,7 +769,7 @@ export async function handleMonthly( existingBreakdown.cacheCreationTokens += breakdown.cacheCreationTokens; existingBreakdown.cacheReadTokens += breakdown.cacheReadTokens; existingBreakdown.cost += breakdown.cost; - existing.modelBreakdowns.set(breakdown.modelName, existingBreakdown); + existing.modelBreakdowns.set(breakdownKey, existingBreakdown); } monthMap.set(month, existing); diff --git a/src/web-server/usage/types.ts b/src/web-server/usage/types.ts index 9348479d..e5588263 100644 --- a/src/web-server/usage/types.ts +++ b/src/web-server/usage/types.ts @@ -12,6 +12,7 @@ /** Per-model token and cost breakdown */ export interface ModelBreakdown { modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index d74bdd56..8e1141a0 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -1,7 +1,10 @@ /** * Unit tests for model-pricing.ts */ -import { describe, it, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, it, expect } from 'bun:test'; import { getModelPricing, calculateCost, @@ -9,6 +12,10 @@ import { hasCustomPricing, type TokenUsage, } from '../../src/web-server/model-pricing'; +import { + clearModelsDevRegistryCache, + setCachedModelsDevRegistry, +} from '../../src/web-server/models-dev/registry-cache'; describe('model-pricing', () => { describe('getModelPricing', () => { @@ -297,4 +304,85 @@ describe('model-pricing', () => { expect(hasCustomPricing('unknown-model-xyz')).toBe(false); }); }); + + describe('models.dev cache integration', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-pricing-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + clearModelsDevRegistryCache(); + setCachedModelsDevRegistry({ + openai: { + id: 'openai', + name: 'OpenAI', + models: { + 'gpt-5.5': { + id: 'gpt-5.5', + name: 'GPT-5.5', + cost: { input: 5, output: 30, cache_read: 0.5 }, + }, + }, + }, + 'github-copilot': { + id: 'github-copilot', + name: 'GitHub Copilot', + models: { + 'gpt-5.5': { + id: 'gpt-5.5', + name: 'GPT-5.5', + cost: { input: 0, output: 0 }, + }, + }, + }, + }); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('resolves provider-prefixed paid API pricing from models.dev', () => { + const pricing = getModelPricing('openai/gpt-5.5'); + expect(pricing.inputPerMillion).toBe(5); + expect(pricing.outputPerMillion).toBe(30); + expect(pricing.cacheReadPerMillion).toBe(0.5); + expect(pricing.cacheCreationPerMillion).toBe(0); + }); + + it('keeps subscription-backed provider pricing distinct from paid API pricing', () => { + const pricing = getModelPricing('gpt-5.5', { provider: 'github-copilot' }); + expect(pricing.inputPerMillion).toBe(0); + expect(pricing.outputPerMillion).toBe(0); + }); + + it('does not use ambiguous model-only models.dev matches', () => { + const pricing = getModelPricing('gpt-5.5'); + expect(pricing).toEqual(getModelPricing('unknown-model-xyz')); + expect(hasCustomPricing('gpt-5.5')).toBe(false); + expect(hasCustomPricing('gpt-5.5', { provider: 'openai' })).toBe(true); + }); + + it('calculates cost with provider-aware models.dev pricing', () => { + const usage: TokenUsage = { + inputTokens: 1_000_000, + outputTokens: 1_000_000, + cacheCreationTokens: 1_000_000, + cacheReadTokens: 1_000_000, + }; + + expect(calculateCost(usage, 'gpt-5.5', { provider: 'openai' })).toBe(35.5); + expect(calculateCost(usage, 'gpt-5.5', { provider: 'ghcp' })).toBe(0); + }); + }); }); diff --git a/tests/unit/models-dev-registry-cache.test.ts b/tests/unit/models-dev-registry-cache.test.ts new file mode 100644 index 00000000..cf640114 --- /dev/null +++ b/tests/unit/models-dev-registry-cache.test.ts @@ -0,0 +1,93 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { getCcsDir } from '../../src/utils/config-manager'; +import { + clearModelsDevRegistryCache, + getCachedModelsDevRegistry, + refreshModelsDevRegistry, + setCachedModelsDevRegistry, +} from '../../src/web-server/models-dev/registry-cache'; + +describe('models.dev registry cache', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-cache-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + clearModelsDevRegistryCache(); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('normalizes and stores provider-keyed models.dev payloads', async () => { + const fetchImpl: typeof fetch = async () => + new Response( + JSON.stringify({ + openai: { + id: 'openai', + name: 'OpenAI', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + ignored: { id: 'ignored' }, + }), + { status: 200 } + ); + + const registry = await refreshModelsDevRegistry({ + force: true, + fetchImpl, + now: () => 123, + }); + + expect(registry?.openai.models?.['gpt-5.5']?.cost?.input).toBe(5); + expect(getCachedModelsDevRegistry({ allowStale: false, now: 123 })?.openai.id).toBe('openai'); + }); + + it('uses stale cache when live refresh fails', async () => { + setCachedModelsDevRegistry( + { + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + }, + 1 + ); + + const fetchImpl: typeof fetch = async () => { + throw new Error('offline'); + }; + + const registry = await refreshModelsDevRegistry({ + force: true, + fetchImpl, + now: () => 1_000_000_000, + }); + + expect(registry?.openai.models?.['gpt-5.5']?.cost?.output).toBe(30); + }); + + it('ignores malformed cache files', () => { + fs.mkdirSync(getCcsDir(), { recursive: true }); + fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json'); + expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); + }); +}); diff --git a/tests/unit/web-server/cliproxy-usage-transformer.test.ts b/tests/unit/web-server/cliproxy-usage-transformer.test.ts index d533865b..8bfd1049 100644 --- a/tests/unit/web-server/cliproxy-usage-transformer.test.ts +++ b/tests/unit/web-server/cliproxy-usage-transformer.test.ts @@ -1,5 +1,12 @@ -import { describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import type { CliproxyUsageApiResponse } from '../../../src/cliproxy/stats-fetcher'; +import { + clearModelsDevRegistryCache, + setCachedModelsDevRegistry, +} from '../../../src/web-server/models-dev/registry-cache'; import { buildCliproxyUsageHistoryAggregates, extractCliproxyUsageHistoryDetails, @@ -101,6 +108,7 @@ describe('cliproxy usage transformer', () => { it('retains failed requests when they carry usage and skips zero-usage failures', () => { const flat = extractCliproxyUsageHistoryDetails(sampleResponse); expect(flat).toHaveLength(4); + expect(flat[0].provider).toBe('gemini'); expect( flat.some( (entry) => @@ -191,4 +199,102 @@ describe('cliproxy usage transformer', () => { expect(monthly[0].outputTokens).toBe(110); expect(monthly[0].cacheReadTokens).toBe(35); }); + + describe('provider-aware pricing', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-cliproxy-models-dev-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + setCachedModelsDevRegistry({ + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30, cache_read: 0.5 } }, + }, + }, + 'github-copilot': { + id: 'github-copilot', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 0, output: 0 } }, + }, + }, + }); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('keeps same model IDs separated by provider in CLIProxy usage', () => { + const response: CliproxyUsageApiResponse = { + usage: { + apis: { + openai: { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T10:00:00.000Z', + source: 'api-account', + auth_index: 0, + tokens: { + input_tokens: 1_000_000, + output_tokens: 1_000_000, + reasoning_tokens: 0, + cached_tokens: 1_000_000, + total_tokens: 3_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + 'github-copilot': { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T11:00:00.000Z', + source: 'copilot-account', + auth_index: 1, + tokens: { + input_tokens: 1_000_000, + output_tokens: 1_000_000, + reasoning_tokens: 0, + cached_tokens: 1_000_000, + total_tokens: 3_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }; + + const [daily] = transformCliproxyToDailyUsage(response); + const paid = daily.modelBreakdowns.find((breakdown) => breakdown.provider === 'openai'); + const subscription = daily.modelBreakdowns.find( + (breakdown) => breakdown.provider === 'github-copilot' + ); + + expect(daily.totalCost).toBe(35.5); + expect(paid?.cost).toBe(35.5); + expect(subscription?.cost).toBe(0); + }); + }); }); From a53862793305a9b66a49be83b9f440ddb7c75283 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 15:25:58 -0400 Subject: [PATCH 2/7] fix(usage): harden models.dev analytics integration --- src/web-server/models-dev/registry-cache.ts | 16 ++++++ src/web-server/usage/aggregator.ts | 50 +++++++++---------- .../usage/cliproxy-usage-transformer.ts | 5 +- src/web-server/usage/data-aggregator.ts | 13 ++--- src/web-server/usage/handlers.ts | 8 +-- src/web-server/usage/model-identity.ts | 37 ++++++++++++++ .../proxy/daemon-lifecycle.test.ts | 2 +- tests/unit/data-aggregator.test.ts | 31 ++++++++++++ tests/unit/models-dev-registry-cache.test.ts | 45 +++++++++++++++++ 9 files changed, 161 insertions(+), 46 deletions(-) create mode 100644 src/web-server/usage/model-identity.ts diff --git a/src/web-server/models-dev/registry-cache.ts b/src/web-server/models-dev/registry-cache.ts index 9a2415d4..9b2a0802 100644 --- a/src/web-server/models-dev/registry-cache.ts +++ b/src/web-server/models-dev/registry-cache.ts @@ -9,6 +9,8 @@ const CACHE_FILE_NAME = 'models-dev-registry-cache.json'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; const LIVE_FETCH_TIMEOUT_MS = 3000; +let pendingBackgroundRefresh: Promise | null = null; + export interface RegistryCacheReadOptions { allowStale?: boolean; now?: number; @@ -131,3 +133,17 @@ export async function refreshModelsDevRegistry( clearTimeout(timeoutId); } } + +export function startModelsDevRegistryRefresh( + options: RegistryRefreshOptions = {} +): Promise { + if (!pendingBackgroundRefresh) { + pendingBackgroundRefresh = refreshModelsDevRegistry(options) + .catch(() => null) + .finally(() => { + pendingBackgroundRefresh = null; + }); + } + + return pendingBackgroundRefresh; +} diff --git a/src/web-server/usage/aggregator.ts b/src/web-server/usage/aggregator.ts index 5496ceb8..44df5b21 100644 --- a/src/web-server/usage/aggregator.ts +++ b/src/web-server/usage/aggregator.ts @@ -34,7 +34,8 @@ import { } from './cliproxy-usage-syncer'; import { scanCodexNativeUsageEntries } from './codex-native-usage-collector'; import { scanDroidNativeUsageEntries } from './droid-native-usage-collector'; -import { refreshModelsDevRegistry } from '../models-dev/registry-cache'; +import { startModelsDevRegistryRefresh } from '../models-dev/registry-cache'; +import { getModelsUsed, getProviderModelKey } from './model-identity'; // ============================================================================ // Multi-Instance Support - Aggregate usage from CCS profiles @@ -112,10 +113,6 @@ function getHourlyRequestCount(hour: HourlyUsage): number { return hour.requestCount ?? hour.modelBreakdowns.length; } -function getModelBreakdownKey(breakdown: { modelName: string; provider?: string }): string { - return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; -} - /** * Merge daily usage data from multiple sources * Combines entries with same date by aggregating tokens @@ -133,14 +130,11 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.cacheCreationTokens += day.cacheCreationTokens; existing.cacheReadTokens += day.cacheReadTokens; existing.totalCost += day.totalCost; - // Merge unique models - const modelSet = new Set([...existing.modelsUsed, ...day.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns by aggregating same modelName for (const breakdown of day.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => getModelBreakdownKey(b) === breakdownKey + (b) => getProviderModelKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -152,12 +146,14 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { // Clone to avoid mutating original + const modelBreakdowns = day.modelBreakdowns.map((b) => ({ ...b })); dateMap.set(day.date, { ...day, - modelsUsed: [...day.modelsUsed], - modelBreakdowns: day.modelBreakdowns.map((b) => ({ ...b })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, }); } } @@ -181,12 +177,10 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { existing.cacheCreationTokens += month.cacheCreationTokens; existing.cacheReadTokens += month.cacheReadTokens; existing.totalCost += month.totalCost; - const modelSet = new Set([...existing.modelsUsed, ...month.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); for (const breakdown of month.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (item) => getModelBreakdownKey(item) === breakdownKey + (item) => getProviderModelKey(item) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -198,11 +192,13 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { + const modelBreakdowns = month.modelBreakdowns.map((breakdown) => ({ ...breakdown })); monthMap.set(month.month, { ...month, - modelsUsed: [...month.modelsUsed], - modelBreakdowns: month.modelBreakdowns.map((breakdown) => ({ ...breakdown })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, }); } } @@ -228,13 +224,11 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.cacheReadTokens += hour.cacheReadTokens; existing.totalCost += hour.totalCost; existing.requestCount = getHourlyRequestCount(existing) + getHourlyRequestCount(hour); - const modelSet = new Set([...existing.modelsUsed, ...hour.modelsUsed]); - existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns for (const breakdown of hour.modelBreakdowns) { - const breakdownKey = getModelBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => getModelBreakdownKey(b) === breakdownKey + (b) => getProviderModelKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -246,11 +240,13 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } + existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { + const modelBreakdowns = hour.modelBreakdowns.map((b) => ({ ...b })); hourMap.set(hour.hour, { ...hour, - modelsUsed: [...hour.modelsUsed], - modelBreakdowns: hour.modelBreakdowns.map((b) => ({ ...b })), + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, requestCount: getHourlyRequestCount(hour), }); } @@ -356,9 +352,9 @@ async function refreshFromSource(): Promise<{ monthly: MonthlyUsage[]; session: SessionUsage[]; }> { - // Refresh model metadata before cost derivation. This is best-effort and - // falls back to stale cache/static pricing when models.dev is unavailable. - await refreshModelsDevRegistry(); + // Keep model metadata warming off the analytics request path. Current + // refreshes use cached/static pricing; the background result helps future runs. + void startModelsDevRegistryRefresh(); // Try to sync CLIProxy snapshot before reading it. // Non-fatal: syncer handles unavailability and stale fallback. diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index f33b8e97..8ee7ac21 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -8,6 +8,7 @@ import type { CliproxyUsageApiResponse, CliproxyRequestDetail } from '../../cliproxy/stats-fetcher'; import { calculateCost } from '../model-pricing'; import type { ModelBreakdown, DailyUsage, HourlyUsage, MonthlyUsage } from './types'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // INTERNAL HELPERS @@ -257,10 +258,6 @@ function sumField(breakdowns: ModelBreakdown[], field: keyof ModelBreakdown): nu return breakdowns.reduce((acc, b) => acc + (b[field] as number), 0); } -function getModelsUsed(breakdowns: ModelBreakdown[]): string[] { - return [...new Set(breakdowns.map((breakdown) => breakdown.modelName))]; -} - // ============================================================================ // TRANSFORMS // ============================================================================ diff --git a/src/web-server/usage/data-aggregator.ts b/src/web-server/usage/data-aggregator.ts index 7640051a..e140ae56 100644 --- a/src/web-server/usage/data-aggregator.ts +++ b/src/web-server/usage/data-aggregator.ts @@ -14,6 +14,7 @@ import { type MonthlyUsage, type SessionUsage, } from './types'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // HELPER FUNCTIONS @@ -91,10 +92,6 @@ function createModelAccumulator(entry: RawUsageEntry): ModelAccumulator { }; } -function getModelsUsed(modelMap: Map): string[] { - return [...new Set(Array.from(modelMap.values()).map((acc) => acc.modelName))]; -} - // ============================================================================ // DAILY AGGREGATION // ============================================================================ @@ -173,7 +170,7 @@ export function aggregateDailyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, }); } @@ -262,7 +259,7 @@ export function aggregateHourlyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, requestCount: hourEntries.length, }); @@ -351,7 +348,7 @@ export function aggregateMonthlyUsage( cacheCreationTokens: totalCacheCreation, cacheReadTokens: totalCacheRead, totalCost, - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, }); } @@ -469,7 +466,7 @@ export function aggregateSessionUsage( totalCost, lastActivity, versions: Array.from(versions), - modelsUsed: getModelsUsed(modelMap), + modelsUsed: getModelsUsed(modelBreakdowns), modelBreakdowns, source, target, diff --git a/src/web-server/usage/handlers.ts b/src/web-server/usage/handlers.ts index d119075b..b88aa37b 100644 --- a/src/web-server/usage/handlers.ts +++ b/src/web-server/usage/handlers.ts @@ -17,6 +17,7 @@ import { getLastFetchTimestamp, refreshUsageCache, } from './aggregator'; +import { getModelsUsed } from './model-identity'; // ============================================================================ // Types @@ -716,7 +717,6 @@ export async function handleMonthly( cacheCreationTokens: number; cacheReadTokens: number; totalCost: number; - modelsUsed: Set; modelBreakdowns: Map< string, { @@ -741,7 +741,6 @@ export async function handleMonthly( cacheCreationTokens: 0, cacheReadTokens: 0, totalCost: 0, - modelsUsed: new Set(), modelBreakdowns: new Map(), }; @@ -750,9 +749,6 @@ export async function handleMonthly( existing.cacheCreationTokens += day.cacheCreationTokens; existing.cacheReadTokens += day.cacheReadTokens; existing.totalCost += day.totalCost; - for (const model of day.modelsUsed) { - existing.modelsUsed.add(model); - } for (const breakdown of day.modelBreakdowns) { const breakdownKey = getBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.get(breakdownKey) ?? { @@ -783,8 +779,8 @@ export async function handleMonthly( cacheCreationTokens: month.cacheCreationTokens, cacheReadTokens: month.cacheReadTokens, totalCost: month.totalCost, - modelsUsed: Array.from(month.modelsUsed), modelBreakdowns: Array.from(month.modelBreakdowns.values()), + modelsUsed: getModelsUsed(Array.from(month.modelBreakdowns.values())), })) .sort((a, b) => a.month.localeCompare(b.month)); } else { diff --git a/src/web-server/usage/model-identity.ts b/src/web-server/usage/model-identity.ts new file mode 100644 index 00000000..bd2e5ecd --- /dev/null +++ b/src/web-server/usage/model-identity.ts @@ -0,0 +1,37 @@ +export interface ProviderModelIdentity { + modelName: string; + provider?: string; +} + +function getProviderKey(provider: string | undefined): string { + return provider?.trim().toLowerCase() ?? ''; +} + +function getModelUsageLabel(item: ProviderModelIdentity, ambiguousModelNames: Set): string { + const provider = getProviderKey(item.provider); + if (provider && ambiguousModelNames.has(item.modelName)) { + return `${provider}/${item.modelName}`; + } + return item.modelName; +} + +export function getProviderModelKey(item: ProviderModelIdentity): string { + return `${getProviderKey(item.provider)}\u0000${item.modelName}`; +} + +export function getModelsUsed(items: ProviderModelIdentity[]): string[] { + const providersByModel = new Map>(); + for (const item of items) { + const providers = providersByModel.get(item.modelName) ?? new Set(); + providers.add(getProviderKey(item.provider)); + providersByModel.set(item.modelName, providers); + } + + const ambiguousModelNames = new Set( + Array.from(providersByModel.entries()) + .filter(([, providers]) => providers.size > 1) + .map(([modelName]) => modelName) + ); + + return [...new Set(items.map((item) => getModelUsageLabel(item, ambiguousModelNames)))]; +} diff --git a/tests/integration/proxy/daemon-lifecycle.test.ts b/tests/integration/proxy/daemon-lifecycle.test.ts index 0d694a3c..403d438f 100644 --- a/tests/integration/proxy/daemon-lifecycle.test.ts +++ b/tests/integration/proxy/daemon-lifecycle.test.ts @@ -395,13 +395,13 @@ describe('openai proxy daemon lifecycle', () => { }); it('keeps the existing proxy running if replacement startup fails', async () => { - const firstPort = await getPort(); const busyServer = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => new Response('busy'), }); const occupiedPort = busyServer.port; + const firstPort = await getPort({ exclude: [occupiedPort] }); try { const settingsPath = path.join(tempDir, 'rollback.settings.json'); diff --git a/tests/unit/data-aggregator.test.ts b/tests/unit/data-aggregator.test.ts index 4984cc15..3f18f185 100644 --- a/tests/unit/data-aggregator.test.ts +++ b/tests/unit/data-aggregator.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from 'bun:test'; import { aggregateDailyUsage, + aggregateHourlyUsage, aggregateMonthlyUsage, aggregateSessionUsage, } from '../../src/web-server/data-aggregator'; @@ -107,6 +108,36 @@ describe('aggregateDailyUsage', () => { const result = aggregateDailyUsage(entries, 'test-source'); expect(result[0].source).toBe('test-source'); }); + + test('keeps provider identity when same model appears under multiple providers', () => { + const entries: RawUsageEntry[] = [ + createEntry({ model: 'gpt-5.5', target: 'openai' }), + createEntry({ model: 'gpt-5.5', target: 'github-copilot' }), + ]; + + const daily = aggregateDailyUsage(entries); + const hourly = aggregateHourlyUsage(entries); + const monthly = aggregateMonthlyUsage(entries); + const session = aggregateSessionUsage(entries); + + expect(daily[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(hourly[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(monthly[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(session[0].modelsUsed).toEqual(['openai/gpt-5.5', 'github-copilot/gpt-5.5']); + expect(daily[0].modelBreakdowns.map((item) => item.provider)).toEqual([ + 'openai', + 'github-copilot', + ]); + }); + + test('preserves model-only modelsUsed entries when provider is unambiguous', () => { + const result = aggregateDailyUsage([ + createEntry({ model: 'gpt-5.5', target: 'openai' }), + createEntry({ model: 'gpt-5.5', target: 'openai' }), + ]); + + expect(result[0].modelsUsed).toEqual(['gpt-5.5']); + }); }); // ============================================================================ diff --git a/tests/unit/models-dev-registry-cache.test.ts b/tests/unit/models-dev-registry-cache.test.ts index cf640114..20b3dfb5 100644 --- a/tests/unit/models-dev-registry-cache.test.ts +++ b/tests/unit/models-dev-registry-cache.test.ts @@ -8,6 +8,7 @@ import { getCachedModelsDevRegistry, refreshModelsDevRegistry, setCachedModelsDevRegistry, + startModelsDevRegistryRefresh, } from '../../src/web-server/models-dev/registry-cache'; describe('models.dev registry cache', () => { @@ -90,4 +91,48 @@ describe('models.dev registry cache', () => { fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json'); expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); }); + + it('starts and coalesces background refreshes without requiring callers to await', async () => { + let fetchCalls = 0; + let resolveResponse: (response: Response) => void = () => undefined; + const responsePromise = new Promise((resolve) => { + resolveResponse = resolve; + }); + const fetchImpl: typeof fetch = async () => { + fetchCalls += 1; + return responsePromise; + }; + + const first = startModelsDevRegistryRefresh({ + force: true, + fetchImpl, + now: () => 456, + }); + const second = startModelsDevRegistryRefresh({ + force: true, + fetchImpl, + now: () => 789, + }); + + expect(first).toBe(second); + expect(fetchCalls).toBe(1); + expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); + + resolveResponse( + new Response( + JSON.stringify({ + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + }), + { status: 200 } + ) + ); + + await first; + expect(getCachedModelsDevRegistry({ allowStale: true })?.openai.id).toBe('openai'); + }); }); From fc90b6f473fd6f78116ab133d75a982debd13f56 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 15:43:13 -0400 Subject: [PATCH 3/7] fix(usage): normalize provider aliases in aggregation --- src/web-server/models-dev/pricing-resolver.ts | 6 ++- src/web-server/usage/data-aggregator.ts | 4 +- src/web-server/usage/model-identity.ts | 8 +++- .../proxy/daemon-lifecycle.test.ts | 46 ++++++++++++++++--- tests/unit/data-aggregator.test.ts | 12 +++++ 5 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/web-server/models-dev/pricing-resolver.ts b/src/web-server/models-dev/pricing-resolver.ts index a45fcd18..26733980 100644 --- a/src/web-server/models-dev/pricing-resolver.ts +++ b/src/web-server/models-dev/pricing-resolver.ts @@ -36,7 +36,9 @@ function normalizeId(value: string): string { return value.trim().toLowerCase(); } -function normalizeProvider(provider: string | undefined): string | undefined { +export function normalizeModelsDevProviderId( + provider: string | null | undefined +): string | undefined { if (!provider) return undefined; const normalized = normalizeId(provider); return PROVIDER_ALIASES[normalized] ?? normalized; @@ -78,7 +80,7 @@ function findProvider( registry: ModelsDevRegistry, provider: string | undefined ): ModelsDevProvider | undefined { - const normalizedProvider = normalizeProvider(provider); + const normalizedProvider = normalizeModelsDevProviderId(provider); if (!normalizedProvider) return undefined; return registry[normalizedProvider]; } diff --git a/src/web-server/usage/data-aggregator.ts b/src/web-server/usage/data-aggregator.ts index e140ae56..db2f5109 100644 --- a/src/web-server/usage/data-aggregator.ts +++ b/src/web-server/usage/data-aggregator.ts @@ -14,7 +14,7 @@ import { type MonthlyUsage, type SessionUsage, } from './types'; -import { getModelsUsed } from './model-identity'; +import { getModelsUsed, normalizeUsageProvider } from './model-identity'; // ============================================================================ // HELPER FUNCTIONS @@ -74,7 +74,7 @@ interface ModelAccumulator { } function getEntryProvider(entry: RawUsageEntry): string | undefined { - return entry.target?.trim().toLowerCase() || undefined; + return normalizeUsageProvider(entry.target); } function getEntryModelKey(entry: RawUsageEntry): string { diff --git a/src/web-server/usage/model-identity.ts b/src/web-server/usage/model-identity.ts index bd2e5ecd..41198a79 100644 --- a/src/web-server/usage/model-identity.ts +++ b/src/web-server/usage/model-identity.ts @@ -1,10 +1,16 @@ +import { normalizeModelsDevProviderId } from '../models-dev/pricing-resolver'; + export interface ProviderModelIdentity { modelName: string; provider?: string; } +export function normalizeUsageProvider(provider: string | undefined): string | undefined { + return normalizeModelsDevProviderId(provider); +} + function getProviderKey(provider: string | undefined): string { - return provider?.trim().toLowerCase() ?? ''; + return normalizeUsageProvider(provider) ?? ''; } function getModelUsageLabel(item: ProviderModelIdentity, ambiguousModelNames: Set): string { diff --git a/tests/integration/proxy/daemon-lifecycle.test.ts b/tests/integration/proxy/daemon-lifecycle.test.ts index 403d438f..6fc5e15f 100644 --- a/tests/integration/proxy/daemon-lifecycle.test.ts +++ b/tests/integration/proxy/daemon-lifecycle.test.ts @@ -12,6 +12,8 @@ import { import { resolveOpenAICompatProxyPreferredPort } from '../../../src/proxy/proxy-port-resolver'; import { resolveOpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; import { + OPENAI_COMPAT_PROXY_ADAPTIVE_PORT_END, + OPENAI_COMPAT_PROXY_ADAPTIVE_PORT_START, getLegacyOpenAICompatProxyPidPath, getLegacyOpenAICompatProxySessionPath, getOpenAICompatProxyPidPath, @@ -38,6 +40,37 @@ afterEach(async () => { fs.rmSync(tempDir, { recursive: true, force: true }); }); +async function findProfileNameWithFreeAdaptivePort(prefix: string): Promise { + for (let index = 0; index < 200; index += 1) { + const profileName = `${prefix}-${index}`; + const preferredPort = resolveOpenAICompatProxyPreferredPort(profileName); + const availablePort = await getPort({ port: preferredPort, host: '127.0.0.1' }); + if (availablePort === preferredPort) { + return profileName; + } + } + + throw new Error(`No free adaptive proxy port found for ${prefix}`); +} + +async function getPortOutsideOpenAICompatAdaptiveRange(): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + const rangeStart = 45_000 + attempt * 101; + const port = await getPort({ + port: getPort.makeRange(rangeStart, rangeStart + 100), + host: '127.0.0.1', + }); + if ( + port < OPENAI_COMPAT_PROXY_ADAPTIVE_PORT_START || + port > OPENAI_COMPAT_PROXY_ADAPTIVE_PORT_END + ) { + return port; + } + } + + throw new Error('No stale proxy fixture port found outside the adaptive range'); +} + describe('openai proxy daemon lifecycle', () => { it('starts, reports status, serves health/models, and stops', async () => { const port = await getPort(); @@ -444,8 +477,9 @@ describe('openai proxy daemon lifecycle', () => { }); it('returns to the adaptive canonical port after a stale fallback session', async () => { - const stalePort = await getPort(); - const settingsPath = path.join(tempDir, 'outside-range.settings.json'); + const profileName = await findProfileNameWithFreeAdaptivePort('outside-range'); + const stalePort = await getPortOutsideOpenAICompatAdaptiveRange(); + const settingsPath = path.join(tempDir, `${profileName}.settings.json`); fs.writeFileSync( settingsPath, JSON.stringify({ @@ -459,7 +493,7 @@ describe('openai proxy daemon lifecycle', () => { 'utf8' ); - const profile = resolveOpenAICompatProfileConfig('outside-range', settingsPath, { + const profile = resolveOpenAICompatProfileConfig(profileName, settingsPath, { ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', ANTHROPIC_AUTH_TOKEN: 'ollama-outside-range', ANTHROPIC_MODEL: 'qwen3-coder', @@ -469,9 +503,9 @@ describe('openai proxy daemon lifecycle', () => { throw new Error('Expected an outside-range OpenAI-compatible profile'); } - fs.mkdirSync(path.dirname(getOpenAICompatProxySessionPath('outside-range')), { recursive: true }); + fs.mkdirSync(path.dirname(getOpenAICompatProxySessionPath(profileName)), { recursive: true }); fs.writeFileSync( - getOpenAICompatProxySessionPath('outside-range'), + getOpenAICompatProxySessionPath(profileName), JSON.stringify( { profileName: profile.profileName, @@ -490,7 +524,7 @@ describe('openai proxy daemon lifecycle', () => { const started = await startOpenAICompatProxy(profile); expect(started.success).toBe(true); - expect(started.port).toBe(resolveOpenAICompatProxyPreferredPort('outside-range')); + expect(started.port).toBe(resolveOpenAICompatProxyPreferredPort(profileName)); expect(started.port).not.toBe(stalePort); }); diff --git a/tests/unit/data-aggregator.test.ts b/tests/unit/data-aggregator.test.ts index 3f18f185..e15d468c 100644 --- a/tests/unit/data-aggregator.test.ts +++ b/tests/unit/data-aggregator.test.ts @@ -138,6 +138,18 @@ describe('aggregateDailyUsage', () => { expect(result[0].modelsUsed).toEqual(['gpt-5.5']); }); + + test('canonicalizes provider aliases before grouping usage', () => { + const result = aggregateDailyUsage([ + createEntry({ model: 'gpt-5.5', target: 'ghcp', inputTokens: 1000 }), + createEntry({ model: 'gpt-5.5', target: 'github-copilot', inputTokens: 2000 }), + ]); + + expect(result[0].modelsUsed).toEqual(['gpt-5.5']); + expect(result[0].modelBreakdowns).toHaveLength(1); + expect(result[0].modelBreakdowns[0].provider).toBe('github-copilot'); + expect(result[0].modelBreakdowns[0].inputTokens).toBe(3000); + }); }); // ============================================================================ From f736190196f02764aa8998275c8b829403b31b7c Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 15:55:35 -0400 Subject: [PATCH 4/7] fix(usage): avoid cross-provider pricing fallback --- src/web-server/models-dev/pricing-resolver.ts | 8 +++++--- tests/unit/model-pricing.test.ts | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/web-server/models-dev/pricing-resolver.ts b/src/web-server/models-dev/pricing-resolver.ts index 26733980..daad1c92 100644 --- a/src/web-server/models-dev/pricing-resolver.ts +++ b/src/web-server/models-dev/pricing-resolver.ts @@ -174,9 +174,11 @@ export function resolveModelsDevPricing( const provider = options.provider ?? prefixed.provider; const modelId = prefixed.model; - return ( - resolveProviderModel(registry, provider, modelId) ?? resolveUnambiguousModel(registry, modelId) - ); + if (provider) { + return resolveProviderModel(registry, provider, modelId); + } + + return resolveUnambiguousModel(registry, modelId); } export function getKnownModelsDevModels(): string[] { diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index 8e1141a0..baa7e11a 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -327,6 +327,11 @@ describe('model-pricing', () => { name: 'GPT-5.5', cost: { input: 5, output: 30, cache_read: 0.5 }, }, + 'openai-exclusive-model': { + id: 'openai-exclusive-model', + name: 'OpenAI Exclusive Model', + cost: { input: 9, output: 18 }, + }, }, }, 'github-copilot': { @@ -373,6 +378,18 @@ describe('model-pricing', () => { expect(hasCustomPricing('gpt-5.5', { provider: 'openai' })).toBe(true); }); + it('does not use another provider pricing when explicit provider lookup misses', () => { + const fallback = getModelPricing('unknown-model-xyz'); + + expect(getModelPricing('openai-exclusive-model', { provider: 'github-copilot' })).toEqual( + fallback + ); + expect(getModelPricing('github-copilot/openai-exclusive-model')).toEqual(fallback); + expect(hasCustomPricing('openai-exclusive-model', { provider: 'github-copilot' })).toBe( + false + ); + }); + it('calculates cost with provider-aware models.dev pricing', () => { const usage: TokenUsage = { inputTokens: 1_000_000, From c7141b3d3af2d23b725efcf1cb02892f5e64ff67 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 16:11:07 -0400 Subject: [PATCH 5/7] fix(usage): honor provider-aware pricing precedence --- src/web-server/model-pricing.ts | 10 +-- .../usage/cliproxy-usage-transformer.ts | 4 +- tests/unit/model-pricing.test.ts | 17 +++++ .../cliproxy-usage-transformer.test.ts | 68 ++++++++++++++++++- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/src/web-server/model-pricing.ts b/src/web-server/model-pricing.ts index e7c7f0c1..4434bb71 100644 --- a/src/web-server/model-pricing.ts +++ b/src/web-server/model-pricing.ts @@ -845,11 +845,6 @@ function hasProviderContext(model: string, options: PricingLookupOptions): boole * first known family tier that happens to share a prefix. */ export function getModelPricing(model: string, options: PricingLookupOptions = {}): ModelPricing { - const exactPricing = PRICING_REGISTRY[model]; - if (exactPricing !== undefined) { - return exactPricing; - } - if (hasProviderContext(model, options)) { const providerPricing = resolveModelsDevPricing(model, options); if (providerPricing !== undefined) { @@ -857,6 +852,11 @@ export function getModelPricing(model: string, options: PricingLookupOptions = { } } + const exactPricing = PRICING_REGISTRY[model]; + if (exactPricing !== undefined) { + return exactPricing; + } + const directOrAliasPricing = getDirectOrAliasPricing(model); if (directOrAliasPricing !== undefined) { return directOrAliasPricing; diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index 8ee7ac21..92ddbfbb 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -8,7 +8,7 @@ import type { CliproxyUsageApiResponse, CliproxyRequestDetail } from '../../cliproxy/stats-fetcher'; import { calculateCost } from '../model-pricing'; import type { ModelBreakdown, DailyUsage, HourlyUsage, MonthlyUsage } from './types'; -import { getModelsUsed } from './model-identity'; +import { getModelsUsed, normalizeUsageProvider } from './model-identity'; // ============================================================================ // INTERNAL HELPERS @@ -60,7 +60,7 @@ function createHistoryDetail( model: string, detail: CliproxyRequestDetail ): CliproxyUsageHistoryDetail { - const pricingProvider = provider.trim().toLowerCase(); + const pricingProvider = normalizeUsageProvider(provider) ?? provider.trim().toLowerCase(); return { model, provider: pricingProvider, diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index baa7e11a..62d3a523 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -327,6 +327,11 @@ describe('model-pricing', () => { name: 'GPT-5.5', cost: { input: 5, output: 30, cache_read: 0.5 }, }, + 'gpt-4o': { + id: 'gpt-4o', + name: 'GPT-4o', + cost: { input: 2.5, output: 10, cache_read: 1.25 }, + }, 'openai-exclusive-model': { id: 'openai-exclusive-model', name: 'OpenAI Exclusive Model', @@ -343,6 +348,11 @@ describe('model-pricing', () => { name: 'GPT-5.5', cost: { input: 0, output: 0 }, }, + 'gpt-4o': { + id: 'gpt-4o', + name: 'GPT-4o', + cost: { input: 0, output: 0 }, + }, }, }, }); @@ -371,6 +381,13 @@ describe('model-pricing', () => { expect(pricing.outputPerMillion).toBe(0); }); + it('prefers provider-aware models.dev pricing over exact static table matches', () => { + const pricing = getModelPricing('gpt-4o', { provider: 'github-copilot' }); + expect(pricing.inputPerMillion).toBe(0); + expect(pricing.outputPerMillion).toBe(0); + expect(getModelPricing('gpt-4o').inputPerMillion).toBe(2.5); + }); + it('does not use ambiguous model-only models.dev matches', () => { const pricing = getModelPricing('gpt-5.5'); expect(pricing).toEqual(getModelPricing('unknown-model-xyz')); diff --git a/tests/unit/web-server/cliproxy-usage-transformer.test.ts b/tests/unit/web-server/cliproxy-usage-transformer.test.ts index 8bfd1049..5c937023 100644 --- a/tests/unit/web-server/cliproxy-usage-transformer.test.ts +++ b/tests/unit/web-server/cliproxy-usage-transformer.test.ts @@ -108,7 +108,7 @@ describe('cliproxy usage transformer', () => { it('retains failed requests when they carry usage and skips zero-usage failures', () => { const flat = extractCliproxyUsageHistoryDetails(sampleResponse); expect(flat).toHaveLength(4); - expect(flat[0].provider).toBe('gemini'); + expect(flat[0].provider).toBe('google'); expect( flat.some( (entry) => @@ -296,5 +296,71 @@ describe('cliproxy usage transformer', () => { expect(paid?.cost).toBe(35.5); expect(subscription?.cost).toBe(0); }); + + it('canonicalizes CLIProxy provider aliases before grouping history details', () => { + const response: CliproxyUsageApiResponse = { + usage: { + apis: { + ghcp: { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T10:00:00.000Z', + source: 'copilot-alias', + auth_index: 0, + tokens: { + input_tokens: 1_000_000, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 1_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + 'github-copilot': { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T11:00:00.000Z', + source: 'copilot-canonical', + auth_index: 1, + tokens: { + input_tokens: 2_000_000, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 2_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }; + + const details = extractCliproxyUsageHistoryDetails(response); + expect(details.map((detail) => detail.provider)).toEqual([ + 'github-copilot', + 'github-copilot', + ]); + + const [daily] = transformCliproxyToDailyUsage(response); + expect(daily.modelBreakdowns).toHaveLength(1); + expect(daily.modelBreakdowns[0]).toMatchObject({ + modelName: 'gpt-5.5', + provider: 'github-copilot', + inputTokens: 3_000_000, + }); + expect(daily.modelsUsed).toEqual(['gpt-5.5']); + }); }); }); From 42fc5281a1e996b3cd0772383187929a610afa34 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 16:26:27 -0400 Subject: [PATCH 6/7] fix(usage): clarify static pricing fallback --- src/web-server/model-pricing.ts | 52 +++++++++++++++++++++----------- tests/unit/model-pricing.test.ts | 10 ++++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/web-server/model-pricing.ts b/src/web-server/model-pricing.ts index 4434bb71..8281b42f 100644 --- a/src/web-server/model-pricing.ts +++ b/src/web-server/model-pricing.ts @@ -758,16 +758,25 @@ const UNKNOWN_MODEL_PRICING: ModelPricing = { // ============================================================================ /** - * Normalize model name for matching - * Handles variations like provider prefixes and case differences + * Strip provider prefixes used by routing/catalog metadata. + * The CCS static table remains model-keyed, so static fallback normalizes + * provider-qualified model IDs before checking aliases. + */ +function stripProviderPrefix(model: string): string { + const trimmed = model.trim(); + const slashIndex = trimmed.indexOf('/'); + if (slashIndex <= 0) { + return trimmed; + } + return trimmed.slice(slashIndex + 1); +} + +/** + * Normalize model name for matching. + * Handles variations like provider prefixes and case differences. */ function normalizeModelName(model: string): string { - // Remove provider prefixes (e.g., "anthropic/claude-..." -> "claude-...") - const normalized = model - .trim() - .toLowerCase() - .replace(/^[^/]+\//, ''); - return normalized; + return stripProviderPrefix(model).toLowerCase(); } /** @@ -835,6 +844,20 @@ function getDirectOrAliasPricing(model: string): ModelPricing | undefined { return undefined; } +function getCcsStaticPricing(model: string): ModelPricing | undefined { + const staticPricing = getDirectOrAliasPricing(model); + if (staticPricing !== undefined) { + return staticPricing; + } + + const providerlessModel = stripProviderPrefix(model); + if (providerlessModel !== model.trim()) { + return getDirectOrAliasPricing(providerlessModel); + } + + return undefined; +} + function hasProviderContext(model: string, options: PricingLookupOptions): boolean { return Boolean(options.provider || /^[^/]+\//.test(model.trim())); } @@ -852,14 +875,9 @@ export function getModelPricing(model: string, options: PricingLookupOptions = { } } - const exactPricing = PRICING_REGISTRY[model]; - if (exactPricing !== undefined) { - return exactPricing; - } - - const directOrAliasPricing = getDirectOrAliasPricing(model); - if (directOrAliasPricing !== undefined) { - return directOrAliasPricing; + const ccsStaticPricing = getCcsStaticPricing(model); + if (ccsStaticPricing !== undefined) { + return ccsStaticPricing; } const modelsDevPricing = resolveModelsDevPricing(model, options); @@ -914,7 +932,7 @@ export function getKnownModels(): string[] { */ export function hasCustomPricing(model: string, options: PricingLookupOptions = {}): boolean { return ( - getDirectOrAliasPricing(model) !== undefined || + getCcsStaticPricing(model) !== undefined || resolveModelsDevPricing(model, options) !== undefined ); } diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index 62d3a523..c304e7f1 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -388,6 +388,16 @@ describe('model-pricing', () => { expect(getModelPricing('gpt-4o').inputPerMillion).toBe(2.5); }); + it('falls back to CCS static pricing when provider-aware models.dev lookup misses a known model', () => { + const staticPricing = getModelPricing('claude-sonnet-4-5'); + + expect(getModelPricing('anthropic/claude-sonnet-4-5')).toEqual(staticPricing); + expect(getModelPricing('claude-sonnet-4-5', { provider: 'anthropic' })).toEqual( + staticPricing + ); + expect(hasCustomPricing('anthropic/claude-sonnet-4-5')).toBe(true); + }); + it('does not use ambiguous model-only models.dev matches', () => { const pricing = getModelPricing('gpt-5.5'); expect(pricing).toEqual(getModelPricing('unknown-model-xyz')); From 30f350a74fac47ec67547fe51325785b1a03ee78 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 16:51:06 -0400 Subject: [PATCH 7/7] fix(usage): preserve pricing overrides and legacy merges --- src/web-server/model-pricing.ts | 29 ++++ src/web-server/usage/aggregator.ts | 48 ++++++- src/web-server/usage/handlers.ts | 39 +++--- src/web-server/usage/model-identity.ts | 51 +++++++ tests/unit/model-pricing.test.ts | 19 +++ ...ge-aggregator-cliproxy-integration.test.ts | 130 ++++++++++++++++++ 6 files changed, 292 insertions(+), 24 deletions(-) diff --git a/src/web-server/model-pricing.ts b/src/web-server/model-pricing.ts index 8281b42f..6d7a8bfa 100644 --- a/src/web-server/model-pricing.ts +++ b/src/web-server/model-pricing.ts @@ -858,6 +858,30 @@ function getCcsStaticPricing(model: string): ModelPricing | undefined { return undefined; } +function getCcsPolicyOverridePricing(model: string): ModelPricing | undefined { + const providerlessModel = stripProviderPrefix(model); + const normalized = normalizeModelName(providerlessModel); + + for (const candidate of getLookupCandidates(providerlessModel)) { + const alias = MODEL_PRICING_ALIASES[candidate]; + if (alias !== undefined) { + const aliasPricing = NORMALIZED_PRICING_REGISTRY[alias]; + if (aliasPricing !== undefined) { + return aliasPricing; + } + } + + if (candidate !== normalized) { + const variantPricing = NORMALIZED_PRICING_REGISTRY[candidate]; + if (variantPricing !== undefined) { + return variantPricing; + } + } + } + + return undefined; +} + function hasProviderContext(model: string, options: PricingLookupOptions): boolean { return Boolean(options.provider || /^[^/]+\//.test(model.trim())); } @@ -869,6 +893,11 @@ function hasProviderContext(model: string, options: PricingLookupOptions): boole */ export function getModelPricing(model: string, options: PricingLookupOptions = {}): ModelPricing { if (hasProviderContext(model, options)) { + const ccsOverridePricing = getCcsPolicyOverridePricing(model); + if (ccsOverridePricing !== undefined) { + return ccsOverridePricing; + } + const providerPricing = resolveModelsDevPricing(model, options); if (providerPricing !== undefined) { return providerPricing.pricing; diff --git a/src/web-server/usage/aggregator.ts b/src/web-server/usage/aggregator.ts index 44df5b21..129f2d73 100644 --- a/src/web-server/usage/aggregator.ts +++ b/src/web-server/usage/aggregator.ts @@ -35,7 +35,11 @@ import { import { scanCodexNativeUsageEntries } from './codex-native-usage-collector'; import { scanDroidNativeUsageEntries } from './droid-native-usage-collector'; import { startModelsDevRegistryRefresh } from '../models-dev/registry-cache'; -import { getModelsUsed, getProviderModelKey } from './model-identity'; +import { + coalesceLegacyProviderlessBreakdowns, + getModelsUsed, + getProviderModelKey, +} from './model-identity'; // ============================================================================ // Multi-Instance Support - Aggregate usage from CCS profiles @@ -113,6 +117,33 @@ function getHourlyRequestCount(hour: HourlyUsage): number { return hour.requestCount ?? hour.modelBreakdowns.length; } +function finalizeDailyUsage(day: DailyUsage): DailyUsage { + const modelBreakdowns = coalesceLegacyProviderlessBreakdowns(day.modelBreakdowns); + return { + ...day, + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, + }; +} + +function finalizeMonthlyUsage(month: MonthlyUsage): MonthlyUsage { + const modelBreakdowns = coalesceLegacyProviderlessBreakdowns(month.modelBreakdowns); + return { + ...month, + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, + }; +} + +function finalizeHourlyUsage(hour: HourlyUsage): HourlyUsage { + const modelBreakdowns = coalesceLegacyProviderlessBreakdowns(hour.modelBreakdowns); + return { + ...hour, + modelsUsed: getModelsUsed(modelBreakdowns), + modelBreakdowns, + }; +} + /** * Merge daily usage data from multiple sources * Combines entries with same date by aggregating tokens @@ -146,7 +177,6 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } - existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { // Clone to avoid mutating original const modelBreakdowns = day.modelBreakdowns.map((b) => ({ ...b })); @@ -159,7 +189,9 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { } } - return Array.from(dateMap.values()).sort((a, b) => a.date.localeCompare(b.date)); + return Array.from(dateMap.values()) + .map(finalizeDailyUsage) + .sort((a, b) => a.date.localeCompare(b.date)); } /** @@ -192,7 +224,6 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } - existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { const modelBreakdowns = month.modelBreakdowns.map((breakdown) => ({ ...breakdown })); monthMap.set(month.month, { @@ -204,7 +235,9 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { } } - return Array.from(monthMap.values()).sort((a, b) => a.month.localeCompare(b.month)); + return Array.from(monthMap.values()) + .map(finalizeMonthlyUsage) + .sort((a, b) => a.month.localeCompare(b.month)); } /** @@ -240,7 +273,6 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.modelBreakdowns.push({ ...breakdown }); } } - existing.modelsUsed = getModelsUsed(existing.modelBreakdowns); } else { const modelBreakdowns = hour.modelBreakdowns.map((b) => ({ ...b })); hourMap.set(hour.hour, { @@ -253,7 +285,9 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { } } - return Array.from(hourMap.values()).sort((a, b) => a.hour.localeCompare(b.hour)); + return Array.from(hourMap.values()) + .map(finalizeHourlyUsage) + .sort((a, b) => a.hour.localeCompare(b.hour)); } /** diff --git a/src/web-server/usage/handlers.ts b/src/web-server/usage/handlers.ts index b88aa37b..5e6e14dc 100644 --- a/src/web-server/usage/handlers.ts +++ b/src/web-server/usage/handlers.ts @@ -17,7 +17,11 @@ import { getLastFetchTimestamp, refreshUsageCache, } from './aggregator'; -import { getModelsUsed } from './model-identity'; +import { + coalesceLegacyProviderlessBreakdowns, + getModelsUsed, + getProviderModelKey, +} from './model-identity'; // ============================================================================ // Types @@ -142,10 +146,6 @@ function calculateUsageTotalTokens( return input + output + cacheCreation + cacheRead; } -function getBreakdownKey(breakdown: { modelName: string; provider?: string }): string { - return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; -} - function parseDateKey(dateString: string): Date { return new Date( Date.UTC( @@ -557,7 +557,7 @@ export async function handleModels( for (const day of filtered) { for (const breakdown of day.modelBreakdowns) { - const modelKey = getBreakdownKey(breakdown); + const modelKey = getProviderModelKey(breakdown); const existing = modelMap.get(modelKey) || { model: breakdown.modelName, provider: breakdown.provider, @@ -750,7 +750,7 @@ export async function handleMonthly( existing.cacheReadTokens += day.cacheReadTokens; existing.totalCost += day.totalCost; for (const breakdown of day.modelBreakdowns) { - const breakdownKey = getBreakdownKey(breakdown); + const breakdownKey = getProviderModelKey(breakdown); const existingBreakdown = existing.modelBreakdowns.get(breakdownKey) ?? { modelName: breakdown.modelName, provider: breakdown.provider, @@ -772,16 +772,21 @@ export async function handleMonthly( } filtered = Array.from(monthMap.values()) - .map((month) => ({ - month: month.month, - inputTokens: month.inputTokens, - outputTokens: month.outputTokens, - cacheCreationTokens: month.cacheCreationTokens, - cacheReadTokens: month.cacheReadTokens, - totalCost: month.totalCost, - modelBreakdowns: Array.from(month.modelBreakdowns.values()), - modelsUsed: getModelsUsed(Array.from(month.modelBreakdowns.values())), - })) + .map((month) => { + const modelBreakdowns = coalesceLegacyProviderlessBreakdowns( + Array.from(month.modelBreakdowns.values()) + ); + return { + month: month.month, + inputTokens: month.inputTokens, + outputTokens: month.outputTokens, + cacheCreationTokens: month.cacheCreationTokens, + cacheReadTokens: month.cacheReadTokens, + totalCost: month.totalCost, + modelBreakdowns, + modelsUsed: getModelsUsed(modelBreakdowns), + }; + }) .sort((a, b) => a.month.localeCompare(b.month)); } else { filtered = await getCachedMonthlyData(); diff --git a/src/web-server/usage/model-identity.ts b/src/web-server/usage/model-identity.ts index 41198a79..7336ac26 100644 --- a/src/web-server/usage/model-identity.ts +++ b/src/web-server/usage/model-identity.ts @@ -5,6 +5,14 @@ export interface ProviderModelIdentity { provider?: string; } +export interface MergeableProviderModelBreakdown extends ProviderModelIdentity { + inputTokens: number; + outputTokens: number; + cacheCreationTokens: number; + cacheReadTokens: number; + cost: number; +} + export function normalizeUsageProvider(provider: string | undefined): string | undefined { return normalizeModelsDevProviderId(provider); } @@ -41,3 +49,46 @@ export function getModelsUsed(items: ProviderModelIdentity[]): string[] { return [...new Set(items.map((item) => getModelUsageLabel(item, ambiguousModelNames)))]; } + +function addBreakdownTokens(target: T, source: T): void { + target.inputTokens += source.inputTokens; + target.outputTokens += source.outputTokens; + target.cacheCreationTokens += source.cacheCreationTokens; + target.cacheReadTokens += source.cacheReadTokens; + target.cost += source.cost; +} + +export function coalesceLegacyProviderlessBreakdowns( + items: T[] +): T[] { + const byModel = new Map(); + for (const item of items) { + const existing = byModel.get(item.modelName) ?? []; + existing.push(item); + byModel.set(item.modelName, existing); + } + + const coalesced: T[] = []; + for (const group of byModel.values()) { + const providerBreakdowns = group.filter((item) => getProviderKey(item.provider)); + const legacyBreakdowns = group.filter((item) => !getProviderKey(item.provider)); + const providerKeys = new Set(providerBreakdowns.map((item) => getProviderKey(item.provider))); + + if (legacyBreakdowns.length > 0 && providerKeys.size === 1 && providerBreakdowns.length > 0) { + const provider = Array.from(providerKeys)[0]; + const [firstProviderBreakdown, ...remainingProviderBreakdowns] = providerBreakdowns; + const merged = { ...firstProviderBreakdown, provider } as T; + + for (const breakdown of [...remainingProviderBreakdowns, ...legacyBreakdowns]) { + addBreakdownTokens(merged, breakdown); + } + + coalesced.push(merged); + continue; + } + + coalesced.push(...group.map((item) => ({ ...item }))); + } + + return coalesced; +} diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index c304e7f1..779dfac9 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -355,6 +355,17 @@ describe('model-pricing', () => { }, }, }, + google: { + id: 'google', + name: 'Google', + models: { + 'gemini-3-flash-preview': { + id: 'gemini-3-flash-preview', + name: 'Gemini 3 Flash Preview', + cost: { input: 99, output: 99 }, + }, + }, + }, }); }); @@ -388,6 +399,14 @@ describe('model-pricing', () => { expect(getModelPricing('gpt-4o').inputPerMillion).toBe(2.5); }); + it('keeps CCS compatibility aliases ahead of provider-aware models.dev matches', () => { + const pricing = getModelPricing('gemini-3-flash-preview', { provider: 'google' }); + const canonical = getModelPricing('gemini-2.5-flash'); + + expect(pricing).toEqual(canonical); + expect(pricing.inputPerMillion).not.toBe(99); + }); + it('falls back to CCS static pricing when provider-aware models.dev lookup misses a known model', () => { const staticPricing = getModelPricing('claude-sonnet-4-5'); diff --git a/tests/unit/web-server/usage-aggregator-cliproxy-integration.test.ts b/tests/unit/web-server/usage-aggregator-cliproxy-integration.test.ts index 55fb2739..fb6ab03f 100644 --- a/tests/unit/web-server/usage-aggregator-cliproxy-integration.test.ts +++ b/tests/unit/web-server/usage-aggregator-cliproxy-integration.test.ts @@ -330,6 +330,136 @@ describe('usage aggregator cliproxy integration', () => { expect(result[0].modelBreakdowns).toHaveLength(2); }); + it('coalesces legacy providerless breakdowns into the only known provider', () => { + const result = aggregator.mergeDailyData([ + [ + { + date: '2026-03-02', + source: 'legacy-cache', + inputTokens: 10, + outputTokens: 1, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 1, + totalCost: 1, + modelsUsed: ['gpt-5.5'], + modelBreakdowns: [ + { + modelName: 'gpt-5.5', + inputTokens: 10, + outputTokens: 1, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 1, + }, + ], + }, + ], + [ + { + date: '2026-03-02', + source: 'cliproxy', + inputTokens: 20, + outputTokens: 2, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 2, + totalCost: 2, + modelsUsed: ['gpt-5.5'], + modelBreakdowns: [ + { + modelName: 'gpt-5.5', + provider: 'openai', + inputTokens: 20, + outputTokens: 2, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 2, + }, + ], + }, + ], + ]); + + expect(result[0].modelBreakdowns).toHaveLength(1); + expect(result[0].modelBreakdowns[0]).toMatchObject({ + modelName: 'gpt-5.5', + provider: 'openai', + inputTokens: 30, + outputTokens: 3, + cost: 3, + }); + expect(result[0].modelsUsed).toEqual(['gpt-5.5']); + }); + + it('keeps legacy providerless breakdowns separate when providers are ambiguous', () => { + const result = aggregator.mergeHourlyData([ + [ + { + hour: '2026-03-02 10:00', + source: 'legacy-cache', + inputTokens: 10, + outputTokens: 1, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 1, + totalCost: 1, + modelsUsed: ['gpt-5.5'], + modelBreakdowns: [ + { + modelName: 'gpt-5.5', + inputTokens: 10, + outputTokens: 1, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 1, + }, + ], + }, + ], + [ + { + hour: '2026-03-02 10:00', + source: 'providers', + inputTokens: 50, + outputTokens: 5, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 5, + totalCost: 5, + modelsUsed: ['openai/gpt-5.5', 'github-copilot/gpt-5.5'], + modelBreakdowns: [ + { + modelName: 'gpt-5.5', + provider: 'openai', + inputTokens: 20, + outputTokens: 2, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 2, + }, + { + modelName: 'gpt-5.5', + provider: 'github-copilot', + inputTokens: 30, + outputTokens: 3, + cacheCreationTokens: 0, + cacheReadTokens: 0, + cost: 3, + }, + ], + }, + ], + ]); + + expect(result[0].modelBreakdowns).toHaveLength(3); + expect(result[0].modelsUsed).toEqual([ + 'gpt-5.5', + 'openai/gpt-5.5', + 'github-copilot/gpt-5.5', + ]); + }); + it('falls back to model cardinality when merging legacy hourly buckets without requestCount', () => { const result = aggregator.mergeHourlyData([ [