From 65adab5bec2814a2f7deb681067892def77b03ad Mon Sep 17 00:00:00 2001 From: "Kai (Tam Nhu) Tran" <61256810+kaitranntt@users.noreply.github.com> Date: Sat, 30 May 2026 14:56:18 -0400 Subject: [PATCH] fix: normalize cliproxy v3 usage snapshots --- src/cliproxy/quota/quota-manager.ts | 10 +- src/management/checks/image-analysis-check.ts | 5 +- src/web-server/usage/cliproxy-usage-syncer.ts | 18 ++- .../usage/cliproxy-usage-transformer.ts | 147 ++++++++++++++++-- .../web-server/cliproxy-usage-syncer.test.ts | 76 ++++++++- .../cliproxy-usage-transformer.test.ts | 59 ++++++- 6 files changed, 279 insertions(+), 36 deletions(-) diff --git a/src/cliproxy/quota/quota-manager.ts b/src/cliproxy/quota/quota-manager.ts index 8574b079..041e2748 100644 --- a/src/cliproxy/quota/quota-manager.ts +++ b/src/cliproxy/quota/quota-manager.ts @@ -483,9 +483,8 @@ export async function reconcileExhaustedRotationAccounts( return []; } - const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import( - '../accounts/account-safety' - ); + const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = + await import('../accounts/account-safety'); restoreExpiredQuotaPauses(); const config = loadOrCreateUnifiedConfig(); @@ -593,9 +592,8 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise * Fix image analysis configuration issues */ export async function fixImageAnalysisConfig(): Promise { - const { updateConfig, loadOrCreateUnifiedConfig } = await import( - '../../config/config-loader-facade' - ); + const { updateConfig, loadOrCreateUnifiedConfig } = + await import('../../config/config-loader-facade'); const config = loadOrCreateUnifiedConfig(); let fixed = false; diff --git a/src/web-server/usage/cliproxy-usage-syncer.ts b/src/web-server/usage/cliproxy-usage-syncer.ts index 83d15a69..74a8d928 100644 --- a/src/web-server/usage/cliproxy-usage-syncer.ts +++ b/src/web-server/usage/cliproxy-usage-syncer.ts @@ -15,6 +15,7 @@ import { buildCliproxyUsageHistoryAggregates, extractCliproxyUsageHistoryDetails, mergeCliproxyUsageHistoryDetails, + normalizeCliproxyUsageHistoryDetail, pruneCliproxyUsageHistoryDetails, type CliproxyUsageHistoryDetail, } from './cliproxy-usage-transformer'; @@ -197,14 +198,27 @@ function readSnapshot(emitWarnings = true): CliproxyUsageSnapshot | null { return null; } - if (!Array.isArray((snapshot as CliproxyUsageSnapshot).details)) { + const details = (snapshot as CliproxyUsageSnapshot).details; + if (!Array.isArray(details)) { if (emitWarnings) { console.log(info('CLIProxy snapshot details missing, will refresh on next sync')); } return null; } - return snapshot as CliproxyUsageSnapshot; + const normalizedDetails = details + .map((detail) => normalizeCliproxyUsageHistoryDetail(detail)) + .filter((detail): detail is CliproxyUsageHistoryDetail => detail !== null); + const { daily, hourly, monthly } = buildCliproxyUsageHistoryAggregates(normalizedDetails); + + return { + version: SNAPSHOT_VERSION, + timestamp: Number(snapshot.timestamp), + details: normalizedDetails, + daily, + hourly, + monthly, + }; } if (snapshot.version === 1 || snapshot.version === 2) { diff --git a/src/web-server/usage/cliproxy-usage-transformer.ts b/src/web-server/usage/cliproxy-usage-transformer.ts index 5f96d632..48e496e3 100644 --- a/src/web-server/usage/cliproxy-usage-transformer.ts +++ b/src/web-server/usage/cliproxy-usage-transformer.ts @@ -64,30 +64,112 @@ function createHistoryDetail( detail: CliproxyRequestDetail ): CliproxyUsageHistoryDetail { const pricingProvider = normalizeUsageProvider(provider) ?? provider.trim().toLowerCase(); + const inputTokens = detail.tokens?.input_tokens ?? 0; + const outputTokens = detail.tokens?.output_tokens ?? 0; + const cacheReadTokens = detail.tokens?.cached_tokens ?? 0; + return { model, provider: pricingProvider, timestamp: detail.timestamp, source: detail.source, authIndex: String(detail.auth_index), - inputTokens: detail.tokens?.input_tokens ?? 0, - outputTokens: detail.tokens?.output_tokens ?? 0, - cacheReadTokens: detail.tokens?.cached_tokens ?? 0, + inputTokens, + outputTokens, + cacheReadTokens, requestCount: 1, - cost: calculateCost( - { - inputTokens: detail.tokens?.input_tokens ?? 0, - outputTokens: detail.tokens?.output_tokens ?? 0, - cacheCreationTokens: 0, - cacheReadTokens: detail.tokens?.cached_tokens ?? 0, - }, + cost: calculateHistoryDetailCost( model, - { provider: pricingProvider } + pricingProvider, + inputTokens, + outputTokens, + cacheReadTokens ), failed: detail.failed, }; } +function calculateHistoryDetailCost( + model: string, + provider: string | undefined, + inputTokens: number, + outputTokens: number, + cacheReadTokens: number +): number { + return calculateCost( + { + inputTokens, + outputTokens, + cacheCreationTokens: 0, + cacheReadTokens, + }, + model, + provider ? { provider } : undefined + ); +} + +function normalizePersistedNumber(value: unknown, fallback = 0): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +function normalizePersistedProvider(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined; + + const trimmed = value.trim(); + if (trimmed.length === 0) return undefined; + + return normalizeUsageProvider(trimmed) ?? trimmed.toLowerCase(); +} + +export function normalizeCliproxyUsageHistoryDetail( + detail: unknown +): CliproxyUsageHistoryDetail | null { + if (!detail || typeof detail !== 'object') return null; + + const candidate = detail as Record; + if ( + typeof candidate.model !== 'string' || + typeof candidate.timestamp !== 'string' || + typeof candidate.source !== 'string' || + !Number.isFinite(Date.parse(candidate.timestamp)) + ) { + return null; + } + + const provider = normalizePersistedProvider(candidate.provider); + const inputTokens = normalizePersistedNumber(candidate.inputTokens); + const outputTokens = normalizePersistedNumber(candidate.outputTokens); + const cacheReadTokens = normalizePersistedNumber(candidate.cacheReadTokens); + const requestCount = Math.max(1, normalizePersistedNumber(candidate.requestCount, 1)); + const cost = normalizePersistedNumber( + candidate.cost, + calculateHistoryDetailCost( + candidate.model, + provider, + inputTokens, + outputTokens, + cacheReadTokens + ) + ); + + return { + model: candidate.model, + ...(provider && { provider }), + timestamp: candidate.timestamp, + source: candidate.source, + authIndex: + typeof candidate.authIndex === 'string' + ? candidate.authIndex + : String(candidate.authIndex ?? ''), + inputTokens, + outputTokens, + cacheReadTokens, + requestCount, + cost, + failed: candidate.failed === true, + }; +} + // ============================================================================ // FLATTEN // ============================================================================ @@ -143,12 +225,53 @@ function createHistorySignature(detail: CliproxyUsageHistoryDetail): string { ].join('|'); } +function createProviderlessHistorySignature(detail: CliproxyUsageHistoryDetail): string { + return [ + detail.model, + detail.timestamp, + detail.source, + detail.authIndex, + detail.inputTokens, + detail.outputTokens, + detail.cacheReadTokens, + detail.requestCount, + detail.failed ? '1' : '0', + ].join('|'); +} + +function hydrateProviderlessHistoryDetails( + existing: CliproxyUsageHistoryDetail[], + incoming: CliproxyUsageHistoryDetail[] +): CliproxyUsageHistoryDetail[] { + const incomingByProviderlessSignature = new Map(); + for (const detail of incoming) { + if (!detail.provider) continue; + + const signature = createProviderlessHistorySignature(detail); + incomingByProviderlessSignature.set(signature, [ + ...(incomingByProviderlessSignature.get(signature) ?? []), + detail, + ]); + } + + return existing.map((detail) => { + if (detail.provider) return detail; + + const matches = incomingByProviderlessSignature.get(createProviderlessHistorySignature(detail)); + const providers = new Set(matches?.map((match) => match.provider).filter(Boolean)); + if (!matches || providers.size !== 1) return detail; + + return { ...detail, provider: matches[0].provider, cost: matches[0].cost }; + }); +} + export function mergeCliproxyUsageHistoryDetails( existing: CliproxyUsageHistoryDetail[], incoming: CliproxyUsageHistoryDetail[] ): CliproxyUsageHistoryDetail[] { + const hydratedExisting = hydrateProviderlessHistoryDetails(existing, incoming); const existingCounts = new Map(); - for (const detail of existing) { + for (const detail of hydratedExisting) { const signature = createHistorySignature(detail); const entry = existingCounts.get(signature); if (entry) { diff --git a/tests/unit/web-server/cliproxy-usage-syncer.test.ts b/tests/unit/web-server/cliproxy-usage-syncer.test.ts index db9dffa1..9f12cb07 100644 --- a/tests/unit/web-server/cliproxy-usage-syncer.test.ts +++ b/tests/unit/web-server/cliproxy-usage-syncer.test.ts @@ -20,7 +20,10 @@ function fetchRawResponse(): Promise { return Promise.resolve(rawResponse); } -function buildResponse(inputTokens: number, timestamp = '2026-03-02T12:00:00.000Z'): CliproxyUsageApiResponse { +function buildResponse( + inputTokens: number, + timestamp = '2026-03-02T12:00:00.000Z' +): CliproxyUsageApiResponse { return { usage: { apis: { @@ -175,6 +178,69 @@ describe('cliproxy usage syncer', () => { }); }); + it('normalizes old v3 snapshot details before loading and merging history', async () => { + await runWithScopedConfigDir(ccsDir, async () => { + const snapshotPath = path.join(ccsDir, 'cache', 'cliproxy-usage', 'latest.json'); + fs.mkdirSync(path.dirname(snapshotPath), { recursive: true }); + fs.writeFileSync( + snapshotPath, + JSON.stringify({ + version: 3, + timestamp: Date.now() - 60_000, + details: [ + { + model: 'gemini-2.5-pro', + timestamp: '2026-03-02T12:00:00.000Z', + source: 'account-a', + authIndex: '0', + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 10, + failed: false, + }, + ], + daily: [ + { + date: '2026-03-02', + source: 'cliproxy', + inputTokens: 100, + outputTokens: 20, + cacheCreationTokens: 0, + cacheReadTokens: 10, + cost: null, + totalCost: null, + modelsUsed: ['gemini-2.5-pro'], + modelBreakdowns: [], + }, + ], + hourly: [], + monthly: [], + }), + 'utf-8' + ); + + const loaded = await loadCachedCliproxyData(); + expect(loaded.daily[0].inputTokens).toBe(100); + expect(Number.isFinite(loaded.daily[0].totalCost)).toBe(true); + expect(loaded.hourly[0].requestCount).toBe(1); + + await syncCliproxyUsage(fetchRawResponse); + + const cached = await loadCachedCliproxyData(); + expect(cached.daily).toHaveLength(1); + expect(cached.daily[0].inputTokens).toBe(100); + expect(cached.hourly[0].requestCount).toBe(1); + + const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf-8')) as { + details: Array<{ requestCount?: number; cost?: number; provider?: string }>; + }; + expect(snapshot.details).toHaveLength(1); + expect(snapshot.details[0].requestCount).toBe(1); + expect(Number.isFinite(snapshot.details[0].cost)).toBe(true); + expect(snapshot.details[0].provider).toBe('google'); + }); + }); + it('migrates legacy v1 and v2 snapshots forward before merging new history', async () => { for (const version of [1, 2]) { await runWithScopedConfigDir(ccsDir, async () => { @@ -264,7 +330,9 @@ describe('cliproxy usage syncer', () => { const cached = await loadCachedCliproxyData(); expect(cached.daily.map((entry) => entry.date)).toEqual(['2026-03-02', '2026-03-01']); - expect(cached.hourly.find((entry) => entry.hour === '2026-03-01 12:00')?.requestCount).toBe(7); + expect(cached.hourly.find((entry) => entry.hour === '2026-03-01 12:00')?.requestCount).toBe( + 7 + ); }); } }); @@ -274,9 +342,7 @@ describe('cliproxy usage syncer', () => { await syncCliproxyUsage(() => Promise.resolve(buildResponse(100, '2024-01-01T12:00:00.000Z')) ); - await syncCliproxyUsage(() => - Promise.resolve(buildResponse(200, new Date().toISOString())) - ); + await syncCliproxyUsage(() => Promise.resolve(buildResponse(200, new Date().toISOString()))); const cached = await loadCachedCliproxyData(); expect(cached.daily.some((entry) => entry.date === '2024-01-01')).toBe(false); diff --git a/tests/unit/web-server/cliproxy-usage-transformer.test.ts b/tests/unit/web-server/cliproxy-usage-transformer.test.ts index 4e590aa6..3bea1553 100644 --- a/tests/unit/web-server/cliproxy-usage-transformer.test.ts +++ b/tests/unit/web-server/cliproxy-usage-transformer.test.ts @@ -11,6 +11,7 @@ import { buildCliproxyUsageHistoryAggregates, extractCliproxyUsageHistoryDetails, mergeCliproxyUsageHistoryDetails, + normalizeCliproxyUsageHistoryDetail, transformCliproxyToDailyUsage, transformCliproxyToHourlyUsage, transformCliproxyToMonthlyUsage, @@ -111,18 +112,12 @@ describe('cliproxy usage transformer', () => { expect(flat[0].provider).toBe('google'); expect( flat.some( - (entry) => - entry.failed === true && - entry.inputTokens === 40 && - entry.outputTokens === 10 + (entry) => entry.failed === true && entry.inputTokens === 40 && entry.outputTokens === 10 ) ).toBe(true); expect( flat.some( - (entry) => - entry.failed === true && - entry.inputTokens === 0 && - entry.outputTokens === 0 + (entry) => entry.failed === true && entry.inputTokens === 0 && entry.outputTokens === 0 ) ).toBe(false); }); @@ -134,6 +129,54 @@ describe('cliproxy usage transformer', () => { expect(merged).toHaveLength(details.length); }); + it('normalizes old v3 history details before merging with provider-aware entries', () => { + const [incoming] = extractCliproxyUsageHistoryDetails({ + usage: { + apis: { + gemini: { + models: { + 'gemini-2.5-pro': { + details: [ + { + timestamp: '2026-03-01T10:15:00.000Z', + source: 'account-a', + auth_index: 0, + tokens: { + input_tokens: 100, + output_tokens: 50, + reasoning_tokens: 0, + cached_tokens: 20, + total_tokens: 170, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }); + const legacyDetail = normalizeCliproxyUsageHistoryDetail({ + model: incoming.model, + timestamp: incoming.timestamp, + source: incoming.source, + authIndex: incoming.authIndex, + inputTokens: incoming.inputTokens, + outputTokens: incoming.outputTokens, + cacheReadTokens: incoming.cacheReadTokens, + failed: incoming.failed, + }); + + expect(legacyDetail?.requestCount).toBe(1); + expect(Number.isFinite(legacyDetail?.cost)).toBe(true); + + const merged = mergeCliproxyUsageHistoryDetails([legacyDetail!], [incoming]); + expect(merged).toHaveLength(1); + expect(merged[0].provider).toBe(incoming.provider); + expect(merged[0].cost).toBe(incoming.cost); + }); + it('preserves legitimate duplicate requests when the incoming batch has more occurrences', () => { const details = extractCliproxyUsageHistoryDetails(sampleResponse); const repeated = [details[0], { ...details[0] }];