From a38c1a75ba73606c68262a16d2e488c30f939da8 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 28 Apr 2026 14:50:26 -0400 Subject: [PATCH] feat: integrate models.dev pricing metadata --- docs/codebase-summary.md | 10 +- docs/project-roadmap.md | 1 + src/web-server/model-pricing.ts | 48 ++++- src/web-server/models-dev/pricing-resolver.ts | 191 ++++++++++++++++++ src/web-server/models-dev/registry-cache.ts | 133 ++++++++++++ src/web-server/models-dev/types.ts | 42 ++++ src/web-server/usage/aggregator.ts | 18 +- .../usage/cliproxy-usage-transformer.ts | 79 +++++--- src/web-server/usage/data-aggregator.ts | 102 +++++----- src/web-server/usage/handlers.ts | 23 ++- src/web-server/usage/types.ts | 1 + tests/unit/model-pricing.test.ts | 90 ++++++++- tests/unit/models-dev-registry-cache.test.ts | 93 +++++++++ .../cliproxy-usage-transformer.test.ts | 108 +++++++++- 14 files changed, 851 insertions(+), 88 deletions(-) create mode 100644 src/web-server/models-dev/pricing-resolver.ts create mode 100644 src/web-server/models-dev/registry-cache.ts create mode 100644 src/web-server/models-dev/types.ts create mode 100644 tests/unit/models-dev-registry-cache.test.ts diff --git a/docs/codebase-summary.md b/docs/codebase-summary.md index cdff9ac7..5d203769 100644 --- a/docs/codebase-summary.md +++ b/docs/codebase-summary.md @@ -2,7 +2,7 @@ Last Updated: 2026-04-26 -Comprehensive overview of the modularized CCS codebase structure following the Phase 9 modularization effort (Settings, Analytics, Auth Monitor splits + Test Infrastructure), v7.1 Remote CLIProxy feature, v7.2 Kiro + GitHub Copilot (ghcp) OAuth providers, v7.14 Hybrid Quota Management, v7.34 Image Analysis Hook, account-context validation hardening, Official Claude Channels runtime support, native Codex runtime target support, and native Codex/Droid usage collectors. +Comprehensive overview of the modularized CCS codebase structure following the Phase 9 modularization effort (Settings, Analytics, Auth Monitor splits + Test Infrastructure), v7.1 Remote CLIProxy feature, v7.2 Kiro + GitHub Copilot (ghcp) OAuth providers, v7.14 Hybrid Quota Management, v7.34 Image Analysis Hook, account-context validation hardening, Official Claude Channels runtime support, native Codex runtime target support, native Codex/Droid usage collectors, and models.dev-backed model pricing metadata. ## Repository Structure @@ -219,9 +219,13 @@ src/ │ ├── codex-native-usage-collector.ts # Native Codex rollout JSONL collector │ ├── droid-native-usage-collector.ts # Native Droid SQLite collector │ └── data-aggregator.ts + ├── models-dev/ # Cached models.dev metadata/pricing registry integration + │ ├── registry-cache.ts + │ ├── pricing-resolver.ts + │ └── types.ts ├── services/ # Shared services │ └── index.ts - └── model-pricing.ts # Model cost definitions (676 lines) + └── model-pricing.ts # Static pricing fallback + models.dev resolver ``` ### Module Categories @@ -550,7 +554,7 @@ ui/src/ | File | Lines | Status | |------|-------|--------| -| model-pricing.ts | 676 | Data file - acceptable | +| model-pricing.ts | 920 | Static pricing fallback and resolver entrypoint | | glmt-proxy.ts | 675 | Legacy internal compatibility path - acceptable for now | | cliproxy-executor.ts | 666 | Core logic - acceptable | | cliproxy-command.ts | 634 | Could split if needed | diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index f439fae7..68d18131 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -81,6 +81,7 @@ All major modularization work is complete. The codebase evolved from monolithic - **#724**: Codex startup is now free-plan safe. CCS defaults new Codex sessions to a cross-plan model and uses runtime fallback handling for unsupported paid-only models without rewriting the saved dashboard settings. - **#737**: Dashboard model pickers in Cursor, Copilot, and CLIProxy now use a searchable combobox with autofocus and explicit no-results states for large model catalogs. - **#736**: `ccs config` now supports explicit dashboard bind hosts via `--host`, and surfaces remote-access warnings plus reachable URLs when the effective bind is non-loopback. +- **#1121**: Usage analytics pricing now refreshes cached models.dev metadata before cost derivation, keeps CCS static pricing as the offline fallback, and carries provider identity through CLIProxy/native runtime breakdowns so subscription-backed providers do not inherit paid API pricing. ### Maintainability Hardening Kickoff diff --git a/src/web-server/model-pricing.ts b/src/web-server/model-pricing.ts index 2c1e6766..e7c7f0c1 100644 --- a/src/web-server/model-pricing.ts +++ b/src/web-server/model-pricing.ts @@ -7,6 +7,12 @@ * All rates are in USD per MILLION tokens. */ +import { + getKnownModelsDevModels, + resolveModelsDevPricing, + type ModelsDevPricingLookupOptions, +} from './models-dev/pricing-resolver'; + // ============================================================================ // TYPE DEFINITIONS // ============================================================================ @@ -25,6 +31,8 @@ export interface TokenUsage { cacheReadTokens: number; } +export type PricingLookupOptions = ModelsDevPricingLookupOptions; + // ============================================================================ // USER-EDITABLE PRICING TABLE // Update rates below (per million tokens in USD) @@ -827,17 +835,38 @@ function getDirectOrAliasPricing(model: string): ModelPricing | undefined { return undefined; } +function hasProviderContext(model: string, options: PricingLookupOptions): boolean { + return Boolean(options.provider || /^[^/]+\//.test(model.trim())); +} + /** * Get pricing for a model with narrow fuzzy matching fallback. * Unknown future model families should fall back instead of inheriting the * first known family tier that happens to share a prefix. */ -export function getModelPricing(model: string): ModelPricing { +export function getModelPricing(model: string, options: PricingLookupOptions = {}): ModelPricing { + const exactPricing = PRICING_REGISTRY[model]; + if (exactPricing !== undefined) { + return exactPricing; + } + + if (hasProviderContext(model, options)) { + const providerPricing = resolveModelsDevPricing(model, options); + if (providerPricing !== undefined) { + return providerPricing.pricing; + } + } + const directOrAliasPricing = getDirectOrAliasPricing(model); if (directOrAliasPricing !== undefined) { return directOrAliasPricing; } + const modelsDevPricing = resolveModelsDevPricing(model, options); + if (modelsDevPricing !== undefined) { + return modelsDevPricing.pricing; + } + for (const candidate of getLookupCandidates(model)) { // Allow provider/routing wrappers to suffix a canonical model ID. for (const [key, pricing] of Object.entries(NORMALIZED_PRICING_REGISTRY)) { @@ -857,8 +886,12 @@ export function getModelPricing(model: string): ModelPricing { * @param model - Model name for pricing lookup * @returns Cost in USD */ -export function calculateCost(usage: TokenUsage, model: string): number { - const pricing = getModelPricing(model); +export function calculateCost( + usage: TokenUsage, + model: string, + options: PricingLookupOptions = {} +): number { + const pricing = getModelPricing(model, options); const inputCost = (usage.inputTokens / 1_000_000) * pricing.inputPerMillion; const outputCost = (usage.outputTokens / 1_000_000) * pricing.outputPerMillion; @@ -873,12 +906,15 @@ export function calculateCost(usage: TokenUsage, model: string): number { * Get list of all known models for UI display */ export function getKnownModels(): string[] { - return Object.keys(PRICING_REGISTRY); + return [...new Set([...Object.keys(PRICING_REGISTRY), ...getKnownModelsDevModels()])]; } /** * Check if a model has custom pricing (not using fallback) */ -export function hasCustomPricing(model: string): boolean { - return getDirectOrAliasPricing(model) !== undefined; +export function hasCustomPricing(model: string, options: PricingLookupOptions = {}): boolean { + return ( + getDirectOrAliasPricing(model) !== undefined || + resolveModelsDevPricing(model, options) !== undefined + ); } diff --git a/src/web-server/models-dev/pricing-resolver.ts b/src/web-server/models-dev/pricing-resolver.ts new file mode 100644 index 00000000..a45fcd18 --- /dev/null +++ b/src/web-server/models-dev/pricing-resolver.ts @@ -0,0 +1,191 @@ +import { getCachedModelsDevRegistry } from './registry-cache'; +import type { ModelsDevCost, ModelsDevModel, ModelsDevProvider, ModelsDevRegistry } from './types'; + +export interface ModelsDevPricing { + inputPerMillion: number; + outputPerMillion: number; + cacheCreationPerMillion: number; + cacheReadPerMillion: number; +} + +export interface ModelsDevPricingResolution { + provider: string; + model: string; + pricing: ModelsDevPricing; +} + +export interface ModelsDevPricingLookupOptions { + provider?: string; +} + +const PROVIDER_ALIASES: Record = { + agy: 'google', + antigravity: 'google', + claude: 'anthropic', + codex: 'openai', + copilot: 'github-copilot', + gemini: 'google', + ghcp: 'github-copilot', + github: 'github-copilot', + kimi: 'moonshotai', + moonshot: 'moonshotai', + qwen: 'alibaba', +}; + +function normalizeId(value: string): string { + return value.trim().toLowerCase(); +} + +function normalizeProvider(provider: string | undefined): string | undefined { + if (!provider) return undefined; + const normalized = normalizeId(provider); + return PROVIDER_ALIASES[normalized] ?? normalized; +} + +function splitProviderPrefix(model: string): { provider?: string; model: string } { + const trimmed = model.trim(); + const slashIndex = trimmed.indexOf('/'); + if (slashIndex <= 0) return { model: trimmed }; + return { + provider: trimmed.slice(0, slashIndex), + model: trimmed.slice(slashIndex + 1), + }; +} + +function stripClaudeDateSuffix(model: string): string { + if (!model.startsWith('claude-')) return model; + return model.replace(/-\d{8}(?=-thinking(?:$|:))/g, '').replace(/-\d{8}(?=$|:)/g, ''); +} + +function getModelCandidates(model: string): string[] { + const normalized = normalizeId(model); + const baseModel = normalized.split(':')[0]; + const candidates = [normalized]; + + if (baseModel !== normalized) candidates.push(baseModel); + + for (const candidate of [...candidates]) { + const stripped = stripClaudeDateSuffix(candidate); + if (stripped !== candidate && !candidates.includes(stripped)) { + candidates.push(stripped); + } + } + + return candidates; +} + +function findProvider( + registry: ModelsDevRegistry, + provider: string | undefined +): ModelsDevProvider | undefined { + const normalizedProvider = normalizeProvider(provider); + if (!normalizedProvider) return undefined; + return registry[normalizedProvider]; +} + +function findModel(provider: ModelsDevProvider, model: string): ModelsDevModel | undefined { + const models = provider.models; + if (!models) return undefined; + + const normalizedEntries = new Map(); + for (const [key, value] of Object.entries(models)) { + normalizedEntries.set(normalizeId(key), value); + if (typeof value.id === 'string') normalizedEntries.set(normalizeId(value.id), value); + } + + for (const candidate of getModelCandidates(model)) { + const match = normalizedEntries.get(candidate); + if (match) return match; + } + + return undefined; +} + +function toNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function toPricing(cost: ModelsDevCost | null | undefined): ModelsDevPricing | undefined { + const input = toNumber(cost?.input); + const output = toNumber(cost?.output); + if (input === undefined || output === undefined) return undefined; + + return { + inputPerMillion: input, + outputPerMillion: output, + cacheCreationPerMillion: toNumber(cost?.cache_write) ?? 0, + cacheReadPerMillion: toNumber(cost?.cache_read) ?? 0, + }; +} + +function samePricing(left: ModelsDevPricing, right: ModelsDevPricing): boolean { + return ( + left.inputPerMillion === right.inputPerMillion && + left.outputPerMillion === right.outputPerMillion && + left.cacheCreationPerMillion === right.cacheCreationPerMillion && + left.cacheReadPerMillion === right.cacheReadPerMillion + ); +} + +function resolveProviderModel( + registry: ModelsDevRegistry, + providerId: string | undefined, + model: string +): ModelsDevPricingResolution | undefined { + const provider = findProvider(registry, providerId); + if (!provider) return undefined; + + const entry = findModel(provider, model); + const pricing = toPricing(entry?.cost); + if (!entry || !pricing) return undefined; + + return { provider: provider.id, model: entry.id, pricing }; +} + +function resolveUnambiguousModel( + registry: ModelsDevRegistry, + model: string +): ModelsDevPricingResolution | undefined { + const matches: ModelsDevPricingResolution[] = []; + + for (const provider of Object.values(registry)) { + const match = resolveProviderModel(registry, provider.id, model); + if (match) matches.push(match); + } + + if (matches.length === 0) return undefined; + const first = matches[0]; + if (matches.every((match) => samePricing(first.pricing, match.pricing))) { + return first; + } + return undefined; +} + +export function resolveModelsDevPricing( + model: string, + options: ModelsDevPricingLookupOptions = {} +): ModelsDevPricingResolution | undefined { + const registry = getCachedModelsDevRegistry({ allowStale: true }); + if (!registry) return undefined; + + const prefixed = splitProviderPrefix(model); + const provider = options.provider ?? prefixed.provider; + const modelId = prefixed.model; + + return ( + resolveProviderModel(registry, provider, modelId) ?? resolveUnambiguousModel(registry, modelId) + ); +} + +export function getKnownModelsDevModels(): string[] { + const registry = getCachedModelsDevRegistry({ allowStale: true }); + if (!registry) return []; + + const ids = new Set(); + for (const provider of Object.values(registry)) { + for (const model of Object.values(provider.models ?? {})) { + if (typeof model.id === 'string') ids.add(`${provider.id}/${model.id}`); + } + } + return Array.from(ids).sort((a, b) => a.localeCompare(b)); +} diff --git a/src/web-server/models-dev/registry-cache.ts b/src/web-server/models-dev/registry-cache.ts new file mode 100644 index 00000000..9a2415d4 --- /dev/null +++ b/src/web-server/models-dev/registry-cache.ts @@ -0,0 +1,133 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { getCcsDir } from '../../utils/config-manager'; +import type { ModelsDevCacheData, ModelsDevProvider, ModelsDevRegistry } from './types'; + +export const MODELS_DEV_API_URL = 'https://models.dev/api.json'; + +const CACHE_FILE_NAME = 'models-dev-registry-cache.json'; +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const LIVE_FETCH_TIMEOUT_MS = 3000; + +export interface RegistryCacheReadOptions { + allowStale?: boolean; + now?: number; +} + +export interface RegistryRefreshOptions { + force?: boolean; + fetchImpl?: typeof fetch; + now?: () => number; +} + +function getCacheFilePath(): string { + return path.join(getCcsDir(), CACHE_FILE_NAME); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function normalizeRegistryPayload(payload: unknown): ModelsDevRegistry | null { + if (!isPlainObject(payload)) return null; + + const providers: ModelsDevRegistry = {}; + for (const [key, value] of Object.entries(payload)) { + if (!isPlainObject(value)) continue; + const id = typeof value.id === 'string' && value.id.trim() ? value.id.trim() : key; + const models = isPlainObject(value.models) + ? (value.models as NonNullable) + : undefined; + if (!models || Object.keys(models).length === 0) continue; + + providers[id] = { + ...(value as ModelsDevProvider), + id, + models, + }; + } + + return Object.keys(providers).length > 0 ? providers : null; +} + +function normalizeCachePayload(payload: unknown): ModelsDevCacheData | null { + if (!isPlainObject(payload)) return null; + if (payload.version !== 1 || typeof payload.fetchedAt !== 'number') return null; + const providers = normalizeRegistryPayload(payload.providers); + return providers ? { version: 1, fetchedAt: payload.fetchedAt, providers } : null; +} + +export function getCachedModelsDevRegistry( + options: RegistryCacheReadOptions = {} +): ModelsDevRegistry | null { + try { + const filePath = getCacheFilePath(); + if (!fs.existsSync(filePath)) return null; + + const cache = normalizeCachePayload(JSON.parse(fs.readFileSync(filePath, 'utf8'))); + if (!cache) return null; + + const now = options.now ?? Date.now(); + if (!options.allowStale && now - cache.fetchedAt > CACHE_TTL_MS) return null; + return cache.providers; + } catch { + return null; + } +} + +export function setCachedModelsDevRegistry( + providers: ModelsDevRegistry, + fetchedAt = Date.now() +): void { + try { + const filePath = getCacheFilePath(); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const cache: ModelsDevCacheData = { version: 1, fetchedAt, providers }; + fs.writeFileSync(filePath, JSON.stringify(cache)); + } catch { + // Best-effort cache writes must not break analytics. + } +} + +export function clearModelsDevRegistryCache(): boolean { + try { + const filePath = getCacheFilePath(); + if (!fs.existsSync(filePath)) return false; + fs.unlinkSync(filePath); + return true; + } catch { + return false; + } +} + +export async function refreshModelsDevRegistry( + options: RegistryRefreshOptions = {} +): Promise { + const now = options.now ?? (() => Date.now()); + if (!options.force) { + const fresh = getCachedModelsDevRegistry({ allowStale: false, now: now() }); + if (fresh) return fresh; + } + + const fetchImpl = options.fetchImpl ?? fetch; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), LIVE_FETCH_TIMEOUT_MS); + + try { + const response = await fetchImpl(MODELS_DEV_API_URL, { + signal: controller.signal, + headers: { Accept: 'application/json' }, + }); + if (!response.ok) return getCachedModelsDevRegistry({ allowStale: true }); + + const providers = normalizeRegistryPayload(await response.json()); + if (!providers) return getCachedModelsDevRegistry({ allowStale: true }); + + setCachedModelsDevRegistry(providers, now()); + return providers; + } catch { + return getCachedModelsDevRegistry({ allowStale: true }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/src/web-server/models-dev/types.ts b/src/web-server/models-dev/types.ts new file mode 100644 index 00000000..8536c7ee --- /dev/null +++ b/src/web-server/models-dev/types.ts @@ -0,0 +1,42 @@ +export interface ModelsDevCost { + input?: number; + output?: number; + cache_read?: number; + cache_write?: number; + [key: string]: unknown; +} + +export interface ModelsDevModel { + id: string; + name?: string; + cost?: ModelsDevCost | null; + limit?: Record; + modalities?: { + input?: string[]; + output?: string[]; + }; + reasoning?: boolean | null; + tool_call?: boolean | null; + structured_output?: boolean | null; + temperature?: boolean | null; + [key: string]: unknown; +} + +export interface ModelsDevProvider { + id: string; + name?: string; + env?: string[]; + npm?: string; + api?: string | null; + doc?: string; + models?: Record; + [key: string]: unknown; +} + +export type ModelsDevRegistry = Record; + +export interface ModelsDevCacheData { + version: 1; + fetchedAt: number; + providers: ModelsDevRegistry; +} diff --git a/src/web-server/usage/aggregator.ts b/src/web-server/usage/aggregator.ts index 97913930..5496ceb8 100644 --- a/src/web-server/usage/aggregator.ts +++ b/src/web-server/usage/aggregator.ts @@ -34,6 +34,7 @@ import { } from './cliproxy-usage-syncer'; import { scanCodexNativeUsageEntries } from './codex-native-usage-collector'; import { scanDroidNativeUsageEntries } from './droid-native-usage-collector'; +import { refreshModelsDevRegistry } from '../models-dev/registry-cache'; // ============================================================================ // Multi-Instance Support - Aggregate usage from CCS profiles @@ -111,6 +112,10 @@ function getHourlyRequestCount(hour: HourlyUsage): number { return hour.requestCount ?? hour.modelBreakdowns.length; } +function getModelBreakdownKey(breakdown: { modelName: string; provider?: string }): string { + return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; +} + /** * Merge daily usage data from multiple sources * Combines entries with same date by aggregating tokens @@ -133,8 +138,9 @@ export function mergeDailyData(sources: DailyUsage[][]): DailyUsage[] { existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns by aggregating same modelName for (const breakdown of day.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => b.modelName === breakdown.modelName + (b) => getModelBreakdownKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -178,8 +184,9 @@ export function mergeMonthlyData(sources: MonthlyUsage[][]): MonthlyUsage[] { const modelSet = new Set([...existing.modelsUsed, ...month.modelsUsed]); existing.modelsUsed = Array.from(modelSet); for (const breakdown of month.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (item) => item.modelName === breakdown.modelName + (item) => getModelBreakdownKey(item) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -225,8 +232,9 @@ export function mergeHourlyData(sources: HourlyUsage[][]): HourlyUsage[] { existing.modelsUsed = Array.from(modelSet); // Merge model breakdowns for (const breakdown of hour.modelBreakdowns) { + const breakdownKey = getModelBreakdownKey(breakdown); const existingBreakdown = existing.modelBreakdowns.find( - (b) => b.modelName === breakdown.modelName + (b) => getModelBreakdownKey(b) === breakdownKey ); if (existingBreakdown) { existingBreakdown.inputTokens += breakdown.inputTokens; @@ -348,6 +356,10 @@ async function refreshFromSource(): Promise<{ monthly: MonthlyUsage[]; session: SessionUsage[]; }> { + // Refresh model metadata before cost derivation. This is best-effort and + // falls back to stale cache/static pricing when models.dev is unavailable. + await refreshModelsDevRegistry(); + // Try to sync CLIProxy snapshot before reading it. // Non-fatal: syncer handles unavailability and stale fallback. await syncCliproxyUsage(); diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index 2ae608e5..f33b8e97 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -16,6 +16,7 @@ import type { ModelBreakdown, DailyUsage, HourlyUsage, MonthlyUsage } from './ty /** Persisted request detail used to rebuild historical CLIProxy analytics buckets */ export interface CliproxyUsageHistoryDetail { model: string; + provider?: string; timestamp: string; source: string; authIndex: string; @@ -36,17 +37,32 @@ interface ModelAccumulator { } /** Build ModelBreakdown from accumulated token counts */ -function buildModelBreakdown(modelName: string, acc: ModelAccumulator): ModelBreakdown { +function buildModelBreakdown( + modelName: string, + provider: string | undefined, + acc: ModelAccumulator +): ModelBreakdown { const { inputTokens, outputTokens, cacheReadTokens, cost } = acc; - return { modelName, inputTokens, outputTokens, cacheCreationTokens: 0, cacheReadTokens, cost }; + return { + modelName, + ...(provider && { provider }), + inputTokens, + outputTokens, + cacheCreationTokens: 0, + cacheReadTokens, + cost, + }; } function createHistoryDetail( + provider: string, model: string, detail: CliproxyRequestDetail ): CliproxyUsageHistoryDetail { + const pricingProvider = provider.trim().toLowerCase(); return { model, + provider: pricingProvider, timestamp: detail.timestamp, source: detail.source, authIndex: String(detail.auth_index), @@ -61,7 +77,8 @@ function createHistoryDetail( cacheCreationTokens: 0, cacheReadTokens: detail.tokens?.cached_tokens ?? 0, }, - model + model, + { provider: pricingProvider } ), failed: detail.failed, }; @@ -92,7 +109,7 @@ export function extractCliproxyUsageHistoryDetails( if (!apis) return []; const results: CliproxyUsageHistoryDetail[] = []; - for (const providerData of Object.values(apis)) { + for (const [provider, providerData] of Object.entries(apis)) { const models = providerData?.models; if (!models) continue; for (const [model, modelData] of Object.entries(models)) { @@ -100,7 +117,7 @@ export function extractCliproxyUsageHistoryDetails( if (!details) continue; for (const detail of details) { if (detail.failed && !hasTrackedUsage(detail)) continue; - results.push(createHistoryDetail(model, detail)); + results.push(createHistoryDetail(provider, model, detail)); } } } @@ -110,6 +127,7 @@ export function extractCliproxyUsageHistoryDetails( function createHistorySignature(detail: CliproxyUsageHistoryDetail): string { return [ detail.model, + detail.provider ?? '', detail.timestamp, detail.source, detail.authIndex, @@ -188,24 +206,35 @@ function aggregateByKey( buildRecord: (key: string, breakdowns: ModelBreakdown[], requestCount: number) => T, sortFn: (a: T, b: T) => number ): T[] { - // bucket: timeKey -> modelName -> accumulator - const buckets = new Map>(); + // bucket: timeKey -> provider/model key -> accumulator + const buckets = new Map< + string, + Map + >(); const requestCounts = new Map(); for (const detail of flat) { const key = keyFn(detail.timestamp); if (!buckets.has(key)) buckets.set(key, new Map()); requestCounts.set(key, (requestCounts.get(key) ?? 0) + detail.requestCount); - const modelMap = buckets.get(key) as Map; - if (!modelMap.has(detail.model)) { - modelMap.set(detail.model, { - inputTokens: 0, - outputTokens: 0, - cacheReadTokens: 0, - cost: 0, + const modelMap = buckets.get(key) as Map< + string, + { modelName: string; provider?: string; acc: ModelAccumulator } + >; + const modelKey = `${detail.provider ?? ''}\u0000${detail.model}`; + if (!modelMap.has(modelKey)) { + modelMap.set(modelKey, { + modelName: detail.model, + provider: detail.provider, + acc: { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cost: 0, + }, }); } - const acc = modelMap.get(detail.model) as ModelAccumulator; + const acc = (modelMap.get(modelKey) as { acc: ModelAccumulator }).acc; acc.inputTokens += detail.inputTokens; acc.outputTokens += detail.outputTokens; acc.cacheReadTokens += detail.cacheReadTokens; @@ -214,8 +243,8 @@ function aggregateByKey( const records: T[] = []; Array.from(buckets.entries()).forEach(([key, modelMap]) => { - const breakdowns = Array.from(modelMap.entries()).map(([name, acc]) => - buildModelBreakdown(name, acc) + const breakdowns = Array.from(modelMap.values()).map((entry) => + buildModelBreakdown(entry.modelName, entry.provider, entry.acc) ); records.push(buildRecord(key, breakdowns, requestCounts.get(key) ?? 0)); }); @@ -228,6 +257,10 @@ function sumField(breakdowns: ModelBreakdown[], field: keyof ModelBreakdown): nu return breakdowns.reduce((acc, b) => acc + (b[field] as number), 0); } +function getModelsUsed(breakdowns: ModelBreakdown[]): string[] { + return [...new Set(breakdowns.map((breakdown) => breakdown.modelName))]; +} + // ============================================================================ // TRANSFORMS // ============================================================================ @@ -249,7 +282,7 @@ export function transformCliproxyToDailyUsage(response: CliproxyUsageApiResponse cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }; }, @@ -278,7 +311,7 @@ export function transformCliproxyToHourlyUsage(response: CliproxyUsageApiRespons cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, requestCount, }; @@ -303,7 +336,7 @@ export function transformCliproxyToMonthlyUsage( cacheCreationTokens: 0, cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), totalCost: sumField(breakdowns, 'cost'), - modelsUsed: breakdowns.map((b) => b.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }), (a, b) => b.month.localeCompare(a.month) @@ -330,7 +363,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }; }, @@ -354,7 +387,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), cost: totalCost, totalCost, - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, requestCount, }; @@ -372,7 +405,7 @@ export function buildCliproxyUsageHistoryAggregates(details: CliproxyUsageHistor cacheCreationTokens: 0, cacheReadTokens: sumField(breakdowns, 'cacheReadTokens'), totalCost: sumField(breakdowns, 'cost'), - modelsUsed: breakdowns.map((breakdown) => breakdown.modelName), + modelsUsed: getModelsUsed(breakdowns), modelBreakdowns: breakdowns, }), (a, b) => b.month.localeCompare(a.month) diff --git a/src/web-server/usage/data-aggregator.ts b/src/web-server/usage/data-aggregator.ts index ba664576..7640051a 100644 --- a/src/web-server/usage/data-aggregator.ts +++ b/src/web-server/usage/data-aggregator.ts @@ -39,6 +39,7 @@ function extractHour(timestamp: string): string { /** Create model breakdown from accumulated data */ function createModelBreakdown( modelName: string, + provider: string | undefined, inputTokens: number, outputTokens: number, cacheCreationTokens: number, @@ -46,11 +47,13 @@ function createModelBreakdown( ): ModelBreakdown { const cost = calculateCost( { inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens }, - modelName + modelName, + { provider } ); return { modelName, + ...(provider && { provider }), inputTokens, outputTokens, cacheCreationTokens, @@ -61,12 +64,37 @@ function createModelBreakdown( /** Accumulator for per-model token counts */ interface ModelAccumulator { + modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; cacheReadTokens: number; } +function getEntryProvider(entry: RawUsageEntry): string | undefined { + return entry.target?.trim().toLowerCase() || undefined; +} + +function getEntryModelKey(entry: RawUsageEntry): string { + return `${getEntryProvider(entry) ?? ''}\u0000${entry.model}`; +} + +function createModelAccumulator(entry: RawUsageEntry): ModelAccumulator { + return { + modelName: entry.model, + provider: getEntryProvider(entry), + inputTokens: 0, + outputTokens: 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, + }; +} + +function getModelsUsed(modelMap: Map): string[] { + return [...new Set(Array.from(modelMap.values()).map((acc) => acc.modelName))]; +} + // ============================================================================ // DAILY AGGREGATION // ============================================================================ @@ -101,19 +129,14 @@ export function aggregateDailyUsage( let totalCacheRead = 0; for (const entry of dateEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -125,9 +148,10 @@ export function aggregateDailyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -149,7 +173,7 @@ export function aggregateDailyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, }); } @@ -194,19 +218,14 @@ export function aggregateHourlyUsage( let totalCacheRead = 0; for (const entry of hourEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -218,9 +237,10 @@ export function aggregateHourlyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -242,7 +262,7 @@ export function aggregateHourlyUsage( cacheReadTokens: totalCacheRead, cost: totalCost, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, requestCount: hourEntries.length, }); @@ -288,19 +308,14 @@ export function aggregateMonthlyUsage( let totalCacheRead = 0; for (const entry of monthEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -312,9 +327,10 @@ export function aggregateMonthlyUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -335,7 +351,7 @@ export function aggregateMonthlyUsage( cacheCreationTokens: totalCacheCreation, cacheReadTokens: totalCacheRead, totalCost, - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, }); } @@ -388,19 +404,14 @@ export function aggregateSessionUsage( let target: string | undefined; for (const entry of orderedEntries) { - const model = entry.model; - const acc = modelMap.get(model) || { - inputTokens: 0, - outputTokens: 0, - cacheCreationTokens: 0, - cacheReadTokens: 0, - }; + const modelKey = getEntryModelKey(entry); + const acc = modelMap.get(modelKey) || createModelAccumulator(entry); acc.inputTokens += entry.inputTokens; acc.outputTokens += entry.outputTokens; acc.cacheCreationTokens += entry.cacheCreationTokens; acc.cacheReadTokens += entry.cacheReadTokens; - modelMap.set(model, acc); + modelMap.set(modelKey, acc); totalInput += entry.inputTokens; totalOutput += entry.outputTokens; @@ -431,9 +442,10 @@ export function aggregateSessionUsage( const modelBreakdowns: ModelBreakdown[] = []; let totalCost = 0; - for (const [modelName, acc] of modelMap) { + for (const acc of modelMap.values()) { const breakdown = createModelBreakdown( - modelName, + acc.modelName, + acc.provider, acc.inputTokens, acc.outputTokens, acc.cacheCreationTokens, @@ -457,7 +469,7 @@ export function aggregateSessionUsage( totalCost, lastActivity, versions: Array.from(versions), - modelsUsed: Array.from(modelMap.keys()), + modelsUsed: getModelsUsed(modelMap), modelBreakdowns, source, target, diff --git a/src/web-server/usage/handlers.ts b/src/web-server/usage/handlers.ts index 2f3bb054..d119075b 100644 --- a/src/web-server/usage/handlers.ts +++ b/src/web-server/usage/handlers.ts @@ -141,6 +141,10 @@ function calculateUsageTotalTokens( return input + output + cacheCreation + cacheRead; } +function getBreakdownKey(breakdown: { modelName: string; provider?: string }): string { + return `${breakdown.provider ?? ''}\u0000${breakdown.modelName}`; +} + function parseDateKey(dateString: string): Date { return new Date( Date.UTC( @@ -204,7 +208,7 @@ export function calculateTokenBreakdownCosts(dailyData: DailyUsage[]): TokenBrea for (const day of dailyData) { for (const breakdown of day.modelBreakdowns) { - const pricing = getModelPricing(breakdown.modelName); + const pricing = getModelPricing(breakdown.modelName, { provider: breakdown.provider }); inputTokens += breakdown.inputTokens; outputTokens += breakdown.outputTokens; cacheCreationTokens += breakdown.cacheCreationTokens; @@ -541,6 +545,7 @@ export async function handleModels( string, { model: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; @@ -551,8 +556,10 @@ export async function handleModels( for (const day of filtered) { for (const breakdown of day.modelBreakdowns) { - const existing = modelMap.get(breakdown.modelName) || { + const modelKey = getBreakdownKey(breakdown); + const existing = modelMap.get(modelKey) || { model: breakdown.modelName, + provider: breakdown.provider, inputTokens: 0, outputTokens: 0, cacheCreationTokens: 0, @@ -564,7 +571,7 @@ export async function handleModels( existing.cacheCreationTokens += breakdown.cacheCreationTokens; existing.cacheReadTokens += breakdown.cacheReadTokens; existing.cost += breakdown.cost; - modelMap.set(breakdown.modelName, existing); + modelMap.set(modelKey, existing); } } @@ -583,7 +590,7 @@ export async function handleModels( const result = models .map((m) => { - const pricing = getModelPricing(m.model); + const pricing = getModelPricing(m.model, { provider: m.provider }); const inputCost = (m.inputTokens / 1_000_000) * pricing.inputPerMillion; const outputCost = (m.outputTokens / 1_000_000) * pricing.outputPerMillion; const cacheCreationCost = @@ -599,6 +606,7 @@ export async function handleModels( return { model: m.model, + provider: m.provider, tokens: totalModelTokens, inputTokens: m.inputTokens, outputTokens: m.outputTokens, @@ -713,6 +721,7 @@ export async function handleMonthly( string, { modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; @@ -745,8 +754,10 @@ export async function handleMonthly( existing.modelsUsed.add(model); } for (const breakdown of day.modelBreakdowns) { - const existingBreakdown = existing.modelBreakdowns.get(breakdown.modelName) ?? { + const breakdownKey = getBreakdownKey(breakdown); + const existingBreakdown = existing.modelBreakdowns.get(breakdownKey) ?? { modelName: breakdown.modelName, + provider: breakdown.provider, inputTokens: 0, outputTokens: 0, cacheCreationTokens: 0, @@ -758,7 +769,7 @@ export async function handleMonthly( existingBreakdown.cacheCreationTokens += breakdown.cacheCreationTokens; existingBreakdown.cacheReadTokens += breakdown.cacheReadTokens; existingBreakdown.cost += breakdown.cost; - existing.modelBreakdowns.set(breakdown.modelName, existingBreakdown); + existing.modelBreakdowns.set(breakdownKey, existingBreakdown); } monthMap.set(month, existing); diff --git a/src/web-server/usage/types.ts b/src/web-server/usage/types.ts index 9348479d..e5588263 100644 --- a/src/web-server/usage/types.ts +++ b/src/web-server/usage/types.ts @@ -12,6 +12,7 @@ /** Per-model token and cost breakdown */ export interface ModelBreakdown { modelName: string; + provider?: string; inputTokens: number; outputTokens: number; cacheCreationTokens: number; diff --git a/tests/unit/model-pricing.test.ts b/tests/unit/model-pricing.test.ts index d74bdd56..8e1141a0 100644 --- a/tests/unit/model-pricing.test.ts +++ b/tests/unit/model-pricing.test.ts @@ -1,7 +1,10 @@ /** * Unit tests for model-pricing.ts */ -import { describe, it, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, it, expect } from 'bun:test'; import { getModelPricing, calculateCost, @@ -9,6 +12,10 @@ import { hasCustomPricing, type TokenUsage, } from '../../src/web-server/model-pricing'; +import { + clearModelsDevRegistryCache, + setCachedModelsDevRegistry, +} from '../../src/web-server/models-dev/registry-cache'; describe('model-pricing', () => { describe('getModelPricing', () => { @@ -297,4 +304,85 @@ describe('model-pricing', () => { expect(hasCustomPricing('unknown-model-xyz')).toBe(false); }); }); + + describe('models.dev cache integration', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-pricing-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + clearModelsDevRegistryCache(); + setCachedModelsDevRegistry({ + openai: { + id: 'openai', + name: 'OpenAI', + models: { + 'gpt-5.5': { + id: 'gpt-5.5', + name: 'GPT-5.5', + cost: { input: 5, output: 30, cache_read: 0.5 }, + }, + }, + }, + 'github-copilot': { + id: 'github-copilot', + name: 'GitHub Copilot', + models: { + 'gpt-5.5': { + id: 'gpt-5.5', + name: 'GPT-5.5', + cost: { input: 0, output: 0 }, + }, + }, + }, + }); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('resolves provider-prefixed paid API pricing from models.dev', () => { + const pricing = getModelPricing('openai/gpt-5.5'); + expect(pricing.inputPerMillion).toBe(5); + expect(pricing.outputPerMillion).toBe(30); + expect(pricing.cacheReadPerMillion).toBe(0.5); + expect(pricing.cacheCreationPerMillion).toBe(0); + }); + + it('keeps subscription-backed provider pricing distinct from paid API pricing', () => { + const pricing = getModelPricing('gpt-5.5', { provider: 'github-copilot' }); + expect(pricing.inputPerMillion).toBe(0); + expect(pricing.outputPerMillion).toBe(0); + }); + + it('does not use ambiguous model-only models.dev matches', () => { + const pricing = getModelPricing('gpt-5.5'); + expect(pricing).toEqual(getModelPricing('unknown-model-xyz')); + expect(hasCustomPricing('gpt-5.5')).toBe(false); + expect(hasCustomPricing('gpt-5.5', { provider: 'openai' })).toBe(true); + }); + + it('calculates cost with provider-aware models.dev pricing', () => { + const usage: TokenUsage = { + inputTokens: 1_000_000, + outputTokens: 1_000_000, + cacheCreationTokens: 1_000_000, + cacheReadTokens: 1_000_000, + }; + + expect(calculateCost(usage, 'gpt-5.5', { provider: 'openai' })).toBe(35.5); + expect(calculateCost(usage, 'gpt-5.5', { provider: 'ghcp' })).toBe(0); + }); + }); }); diff --git a/tests/unit/models-dev-registry-cache.test.ts b/tests/unit/models-dev-registry-cache.test.ts new file mode 100644 index 00000000..cf640114 --- /dev/null +++ b/tests/unit/models-dev-registry-cache.test.ts @@ -0,0 +1,93 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { getCcsDir } from '../../src/utils/config-manager'; +import { + clearModelsDevRegistryCache, + getCachedModelsDevRegistry, + refreshModelsDevRegistry, + setCachedModelsDevRegistry, +} from '../../src/web-server/models-dev/registry-cache'; + +describe('models.dev registry cache', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-cache-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + clearModelsDevRegistryCache(); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('normalizes and stores provider-keyed models.dev payloads', async () => { + const fetchImpl: typeof fetch = async () => + new Response( + JSON.stringify({ + openai: { + id: 'openai', + name: 'OpenAI', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + ignored: { id: 'ignored' }, + }), + { status: 200 } + ); + + const registry = await refreshModelsDevRegistry({ + force: true, + fetchImpl, + now: () => 123, + }); + + expect(registry?.openai.models?.['gpt-5.5']?.cost?.input).toBe(5); + expect(getCachedModelsDevRegistry({ allowStale: false, now: 123 })?.openai.id).toBe('openai'); + }); + + it('uses stale cache when live refresh fails', async () => { + setCachedModelsDevRegistry( + { + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } }, + }, + }, + }, + 1 + ); + + const fetchImpl: typeof fetch = async () => { + throw new Error('offline'); + }; + + const registry = await refreshModelsDevRegistry({ + force: true, + fetchImpl, + now: () => 1_000_000_000, + }); + + expect(registry?.openai.models?.['gpt-5.5']?.cost?.output).toBe(30); + }); + + it('ignores malformed cache files', () => { + fs.mkdirSync(getCcsDir(), { recursive: true }); + fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json'); + expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull(); + }); +}); diff --git a/tests/unit/web-server/cliproxy-usage-transformer.test.ts b/tests/unit/web-server/cliproxy-usage-transformer.test.ts index d533865b..8bfd1049 100644 --- a/tests/unit/web-server/cliproxy-usage-transformer.test.ts +++ b/tests/unit/web-server/cliproxy-usage-transformer.test.ts @@ -1,5 +1,12 @@ -import { describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import type { CliproxyUsageApiResponse } from '../../../src/cliproxy/stats-fetcher'; +import { + clearModelsDevRegistryCache, + setCachedModelsDevRegistry, +} from '../../../src/web-server/models-dev/registry-cache'; import { buildCliproxyUsageHistoryAggregates, extractCliproxyUsageHistoryDetails, @@ -101,6 +108,7 @@ describe('cliproxy usage transformer', () => { it('retains failed requests when they carry usage and skips zero-usage failures', () => { const flat = extractCliproxyUsageHistoryDetails(sampleResponse); expect(flat).toHaveLength(4); + expect(flat[0].provider).toBe('gemini'); expect( flat.some( (entry) => @@ -191,4 +199,102 @@ describe('cliproxy usage transformer', () => { expect(monthly[0].outputTokens).toBe(110); expect(monthly[0].cacheReadTokens).toBe(35); }); + + describe('provider-aware pricing', () => { + let tempRoot = ''; + let originalCcsHome: string | undefined; + let originalCcsDir: string | undefined; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-cliproxy-models-dev-')); + originalCcsHome = process.env.CCS_HOME; + originalCcsDir = process.env.CCS_DIR; + process.env.CCS_HOME = tempRoot; + delete process.env.CCS_DIR; + setCachedModelsDevRegistry({ + openai: { + id: 'openai', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30, cache_read: 0.5 } }, + }, + }, + 'github-copilot': { + id: 'github-copilot', + models: { + 'gpt-5.5': { id: 'gpt-5.5', cost: { input: 0, output: 0 } }, + }, + }, + }); + }); + + afterEach(() => { + clearModelsDevRegistryCache(); + if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; + else delete process.env.CCS_HOME; + if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir; + else delete process.env.CCS_DIR; + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('keeps same model IDs separated by provider in CLIProxy usage', () => { + const response: CliproxyUsageApiResponse = { + usage: { + apis: { + openai: { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T10:00:00.000Z', + source: 'api-account', + auth_index: 0, + tokens: { + input_tokens: 1_000_000, + output_tokens: 1_000_000, + reasoning_tokens: 0, + cached_tokens: 1_000_000, + total_tokens: 3_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + 'github-copilot': { + models: { + 'gpt-5.5': { + details: [ + { + timestamp: '2026-03-03T11:00:00.000Z', + source: 'copilot-account', + auth_index: 1, + tokens: { + input_tokens: 1_000_000, + output_tokens: 1_000_000, + reasoning_tokens: 0, + cached_tokens: 1_000_000, + total_tokens: 3_000_000, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }; + + const [daily] = transformCliproxyToDailyUsage(response); + const paid = daily.modelBreakdowns.find((breakdown) => breakdown.provider === 'openai'); + const subscription = daily.modelBreakdowns.find( + (breakdown) => breakdown.provider === 'github-copilot' + ); + + expect(daily.totalCost).toBe(35.5); + expect(paid?.cost).toBe(35.5); + expect(subscription?.cost).toBe(0); + }); + }); });