diff --git a/src/cliproxy/services/__tests__/stats-fetcher.test.ts b/src/cliproxy/services/__tests__/stats-fetcher.test.ts new file mode 100644 index 00000000..b6e59769 --- /dev/null +++ b/src/cliproxy/services/__tests__/stats-fetcher.test.ts @@ -0,0 +1,1436 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { runWithScopedConfigDir } from '../../../utils/config-manager'; +import { __testExports, fetchCliproxyStats, fetchCliproxyUsageRaw } from '../stats-fetcher'; +import { mergeUsageResponseWithMissingDetails } from '../usage-compatibility-transformer'; + +const originalFetch = globalThis.fetch; +const originalDateNow = Date.now; + +let ccsDir = ''; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +function requestUrl(input: RequestInfo | URL): string { + return input instanceof Request ? input.url : String(input); +} + +function createCodexQueueRecord() { + return { + timestamp: '2026-05-05T18:45:00.000Z', + provider: 'codex', + model: 'gpt-5.5', + alias: 'gpt-5.5', + source: 'user@example.com', + auth_index: 'codex-auth', + tokens: { + input_tokens: 12, + output_tokens: 8, + reasoning_tokens: 0, + cached_tokens: 3, + total_tokens: 23, + }, + failed: false, + }; +} + +function localTimestampIso(datePart: string, timePart: string): string { + const [year, month, day] = datePart.split('-').map(Number); + const [hour, minute, second] = timePart.split(':').map(Number); + return new Date(year, month - 1, day, hour, minute, second).toISOString(); +} + +function writeCliproxyMainLog(lines: string[]): void { + const logsDir = path.join(ccsDir, 'cliproxy', 'logs'); + fs.mkdirSync(logsDir, { recursive: true }); + fs.writeFileSync(path.join(logsDir, 'main.log'), `${lines.join('\n')}\n`, 'utf-8'); +} + +beforeEach(() => { + ccsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-stats-fetcher-')); + __testExports.clearCachedUsageQueueResponse(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + Date.now = originalDateNow; + __testExports.clearCachedUsageQueueResponse(); + fs.rmSync(ccsDir, { recursive: true, force: true }); +}); + +describe('fetchCliproxyUsageRaw', () => { + it('falls back to CLIProxy usage-queue when the legacy aggregate endpoint is unavailable', async () => { + const requestedUrls: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + requestedUrls.push(url); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([createCodexQueueRecord()]); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19201)); + + expect(requestedUrls.some((url) => url.endsWith('/v0/management/usage'))).toBe(true); + expect(requestedUrls.some((url) => url.includes('/v0/management/usage-queue'))).toBe(true); + expect(raw?.usage?.total_requests).toBe(1); + expect(raw?.usage?.success_count).toBe(1); + expect(raw?.usage?.total_tokens).toBe(23); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details?.[0]).toMatchObject({ + source: 'user@example.com', + auth_index: 'codex-auth', + failed: false, + }); + }); + + it('uses queue details to enrich successful legacy aggregate usage without details', async () => { + const requestedUrls: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + requestedUrls.push(url); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ + failed_requests: 0, + usage: { + total_requests: 3, + success_count: 3, + failure_count: 0, + total_tokens: 69, + apis: { + codex: { + total_requests: 3, + total_tokens: 69, + models: {}, + }, + }, + }, + }); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([createCodexQueueRecord()]); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19215)); + + expect(requestedUrls.some((url) => url.endsWith('/v0/management/usage'))).toBe(true); + expect(requestedUrls.some((url) => url.includes('/v0/management/usage-queue'))).toBe(true); + expect(raw?.usage?.total_requests).toBe(3); + expect(raw?.usage?.success_count).toBe(3); + expect(raw?.usage?.total_tokens).toBe(69); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(1); + }); + + it('drains CLIProxy usage-queue batches until the queue is exhausted', async () => { + let queueCalls = 0; + const firstBatch = Array.from({ length: 1000 }, (_, index) => ({ + ...createCodexQueueRecord(), + timestamp: `2026-05-05T18:${String(index % 60).padStart(2, '0')}:00.000Z`, + source: `oauth|codex-user-${index}@example.com-pro.json`, + auth_index: `codex-auth-${index}`, + })); + const finalBatch = [ + { + ...createCodexQueueRecord(), + timestamp: '2026-05-05T19:45:00.000Z', + source: 'oauth|codex-final@example.com-pro.json', + auth_index: 'codex-auth-final', + }, + ]; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + return jsonResponse(queueCalls === 1 ? firstBatch : finalBatch); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19212)); + + expect(queueCalls).toBe(2); + expect(raw?.usage?.total_requests).toBe(1001); + expect(raw?.usage?.total_tokens).toBe(23 * 1001); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(1001); + }); + + it('falls back instead of reporting a repeated full queue batch as complete', async () => { + let queueCalls = 0; + let apiKeyUsageCalls = 0; + const fullBatch = Array.from({ length: 1000 }, (_, index) => ({ + ...createCodexQueueRecord(), + timestamp: `2026-05-05T18:${String(index % 60).padStart(2, '0')}:00.000Z`, + source: `oauth|codex-user-${index}@example.com-pro.json`, + auth_index: `codex-auth-${index}`, + })); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + return jsonResponse(fullBatch); + } + if (url.endsWith('/v0/management/api-key-usage')) { + apiKeyUsageCalls++; + return jsonResponse({ + codex: { + 'oauth|codex-user@example.com-pro.json': { + success: 3, + failed: 0, + }, + }, + }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19213)); + + expect(queueCalls).toBe(2); + expect(apiKeyUsageCalls).toBe(1); + expect(raw?.usage?.total_requests).toBe(3); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(3); + }); + + it('falls back instead of reporting a transiently failed queue drain as complete', async () => { + let queueCalls = 0; + let apiKeyUsageCalls = 0; + const fullBatch = Array.from({ length: 1000 }, (_, index) => ({ + ...createCodexQueueRecord(), + timestamp: `2026-05-05T18:${String(index % 60).padStart(2, '0')}:00.000Z`, + source: `oauth|codex-user-${index}@example.com-pro.json`, + auth_index: `codex-auth-${index}`, + })); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + if (queueCalls === 1) { + return jsonResponse(fullBatch); + } + throw new Error('temporary queue read failure'); + } + if (url.endsWith('/v0/management/api-key-usage')) { + apiKeyUsageCalls++; + return jsonResponse({ + codex: { + 'oauth|codex-user@example.com-pro.json': { + success: 4, + failed: 1, + }, + }, + }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19214)); + + expect(queueCalls).toBe(2); + expect(apiKeyUsageCalls).toBe(1); + expect(raw?.usage?.total_requests).toBe(5); + expect(raw?.usage?.success_count).toBe(4); + expect(raw?.usage?.failure_count).toBe(1); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(5); + }); + + it('falls back when usage queue draining exceeds the overall deadline', async () => { + let queueCalls = 0; + let apiKeyUsageCalls = 0; + Date.now = (() => (queueCalls === 0 ? 0 : 31_000)) as typeof Date.now; + const fullBatch = Array.from({ length: 1000 }, (_, index) => ({ + ...createCodexQueueRecord(), + timestamp: `2026-05-05T18:${String(index % 60).padStart(2, '0')}:00.000Z`, + source: `user-${index}@example.com`, + auth_index: `codex-auth-${index}`, + })); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + return jsonResponse(fullBatch); + } + if (url.endsWith('/v0/management/api-key-usage')) { + apiKeyUsageCalls++; + return jsonResponse({ + codex: { + 'oauth|codex-user@example.com-pro.json': { + success: 6, + failed: 0, + }, + }, + }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19217)); + + expect(queueCalls).toBe(1); + expect(apiKeyUsageCalls).toBe(1); + expect(raw?.usage?.total_requests).toBe(6); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(6); + }); + + it('keeps queue stats available after the same CLIProxy usage queue has been drained', async () => { + let queueCalls = 0; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + return jsonResponse(queueCalls === 1 ? [createCodexQueueRecord()] : []); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const firstRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19204)); + const secondRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19204)); + + expect(firstRaw?.usage?.total_requests).toBe(1); + expect(secondRaw?.usage?.total_requests).toBe(1); + expect(secondRaw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(1); + }); + + it('uses fresh API-key totals before cached queue details after the queue drains', async () => { + let queueCalls = 0; + let apiKeyUsageCalls = 0; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + queueCalls++; + return jsonResponse(queueCalls === 1 ? [createCodexQueueRecord()] : []); + } + if (url.endsWith('/v0/management/api-key-usage')) { + apiKeyUsageCalls++; + return jsonResponse({ + codex: { + 'oauth|codex-user@example.com-pro.json': { + success: 5, + failed: 0, + }, + }, + }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const firstRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19216)); + const secondRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19216)); + + expect(firstRaw?.usage?.total_requests).toBe(1); + expect(apiKeyUsageCalls).toBe(1); + expect(secondRaw?.usage?.total_requests).toBe(5); + expect(secondRaw?.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(1); + }); + + it('does not reuse drained usage-queue stats for a different CLIProxy management URL', async () => { + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse(url.includes(':19205') ? [createCodexQueueRecord()] : []); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const firstRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19205)); + const secondRaw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19206)); + + expect(firstRaw?.usage?.total_requests).toBe(1); + expect(secondRaw?.usage?.total_requests).toBe(0); + expect(secondRaw?.usage?.apis).toEqual({}); + }); + + it('uses API-key usage totals when neither aggregate nor queue usage is available', async () => { + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ + openai: { + 'https://api.example.test|sk-redacted': { + success: 2, + failed: 1, + }, + }, + }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19202)); + + expect(raw?.usage?.total_requests).toBe(3); + expect(raw?.usage?.success_count).toBe(2); + expect(raw?.usage?.failure_count).toBe(1); + expect(raw?.usage?.apis?.openai.total_requests).toBe(3); + expect(raw?.usage?.apis?.openai.models).toEqual({}); + }); + + it('merges local OAuth log usage when management endpoints have only aggregate API-key totals', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ + openai: { + 'https://api.example.test|sk-redacted': { + success: 2, + failed: 0, + }, + }, + }); + } + if (url.endsWith('/v0/management/auth-files')) { + return jsonResponse({ files: [] }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const stats = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyStats(19207)); + + expect(stats?.totalRequests).toBe(3); + expect(stats?.successCount).toBe(3); + expect(stats?.requestsByProvider).toEqual({ openai: 2, codex: 1 }); + expect(stats?.requestsByModel).toEqual({ 'gpt-5.5': 1 }); + expect(stats?.accountStats['codex:user@example.com']).toMatchObject({ + provider: 'codex', + source: 'user@example.com', + successCount: 1, + failureCount: 0, + }); + }); + + it('adds missing OAuth details to matching API-key provider totals without double-counting', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ + codex: { + 'oauth|codex-user@example.com-pro.json': { + success: 1, + failed: 0, + }, + }, + }); + } + if (url.endsWith('/v0/management/auth-files')) { + return jsonResponse({ files: [] }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const stats = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyStats(19209)); + + expect(stats?.totalRequests).toBe(1); + expect(stats?.successCount).toBe(1); + expect(stats?.requestsByProvider).toEqual({ codex: 1 }); + expect(stats?.requestsByModel).toEqual({ 'gpt-5.5': 1 }); + expect(stats?.accountStats['codex:user@example.com']).toMatchObject({ + provider: 'codex', + source: 'user@example.com', + successCount: 1, + failureCount: 0, + }); + }); + + it('keeps log-derived OAuth details when the same provider already has other details', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 7, + apis: { + codex: { + total_requests: 1, + total_tokens: 7, + models: { + 'gpt-4o': { + total_requests: 1, + total_tokens: 7, + details: [ + { + timestamp: '2026-05-05T18:44:00.000Z', + source: 'api-key|sk-redacted', + auth_index: 'api-key|sk-redacted', + tokens: { + input_tokens: 4, + output_tokens: 3, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 7, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }); + } + if (url.endsWith('/v0/management/auth-files')) { + return jsonResponse({ files: [] }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const stats = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyStats(19210)); + + expect(stats?.totalRequests).toBe(2); + expect(stats?.successCount).toBe(2); + expect(stats?.requestsByProvider).toEqual({ codex: 2 }); + expect(stats?.requestsByModel).toEqual({ 'gpt-4o': 1, 'gpt-5.5': 1 }); + expect(stats?.accountStats['codex:user@example.com']).toMatchObject({ + provider: 'codex', + source: 'user@example.com', + successCount: 1, + failureCount: 0, + }); + }); + + it('does not double-count local OAuth logs when usage queue already has provider details', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([ + { + ...createCodexQueueRecord(), + source: 'user@example.com', + auth_index: 'codex-auth', + }, + ]); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19208)); + + expect(raw?.usage?.total_requests).toBe(1); + expect(raw?.usage?.total_tokens).toBe(23); + expect(raw?.usage?.apis?.codex.total_requests).toBe(1); + const details = raw?.usage?.apis?.codex.models?.['gpt-5.5'].details; + expect(details).toHaveLength(1); + expect(details?.[0]?.tokens.total_tokens).toBe(23); + }); + + it('scans the full CLIProxy main log when OAuth selection and completion are far apart', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + 'x'.repeat(1024 * 1024 + 64), + '2026-05-05T18:45:01.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19211)); + + expect(raw?.usage?.total_requests).toBe(1); + expect(raw?.usage?.apis?.codex.models?.['gpt-5.5'].details?.[0]).toMatchObject({ + auth_index: 'codex-user@example.com-pro.json', + failed: false, + }); + }); + + it('parses bracketed CLIProxy main.log request ids and local timestamps', async () => { + writeCliproxyMainLog([ + '[2026-05-05 18:45:00] [req-a] [debug] Use OAuth provider=codex auth_file=codex-user-a@example.com-pro.json for model gpt-5.5', + '[2026-05-05 18:45:00] [req-b] [debug] Use OAuth provider=codex auth_file=codex-user-b@example.com-pro.json for model gpt-5.5', + '[2026-05-05 18:45:01] [req-b] [info ] POST "/api/provider/codex/v1/messages?beta=true" status=200', + '[2026-05-05 18:45:02] [req-a] [info ] POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19218)); + const details = raw?.usage?.apis?.codex.models?.['gpt-5.5'].details; + + expect(details?.map((detail) => detail.auth_index)).toEqual([ + 'codex-user-b@example.com-pro.json', + 'codex-user-a@example.com-pro.json', + ]); + expect(details?.map((detail) => detail.request_id)).toEqual(['req-b', 'req-a']); + expect(details?.map((detail) => detail.timestamp)).toEqual([ + localTimestampIso('2026-05-05', '18:45:01'), + localTimestampIso('2026-05-05', '18:45:02'), + ]); + }); + + it('removes request-id pending entries when FIFO log matching consumes them', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-user@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO POST "/api/provider/codex/v1/messages?beta=true" status=200', + '2026-05-05T18:45:02.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19219)); + const details = raw?.usage?.apis?.codex.models?.['gpt-5.5'].details; + + expect(details).toHaveLength(1); + expect(details?.[0]?.request_id).toBe('req-1'); + }); + + it('removes overwritten request-id pending entries from provider FIFO queues', async () => { + writeCliproxyMainLog([ + '2026-05-05T18:45:00.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-old@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:01.000Z INFO request_id=req-1 Use OAuth provider=codex auth_file=codex-new@example.com-pro.json for model gpt-5.5', + '2026-05-05T18:45:02.000Z INFO request_id=req-1 POST "/api/provider/codex/v1/messages?beta=true" status=200', + '2026-05-05T18:45:03.000Z INFO POST "/api/provider/codex/v1/messages?beta=true" status=200', + ]); + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([]); + } + if (url.endsWith('/v0/management/api-key-usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const raw = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyUsageRaw(19220)); + const details = raw?.usage?.apis?.codex.models?.['gpt-5.5'].details; + + expect(details).toHaveLength(1); + expect(details?.[0]?.auth_index).toBe('codex-new@example.com-pro.json'); + }); + + it('does not inflate token totals when enriching already-counted aggregate requests', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 11, + apis: { + codex: { + total_requests: 1, + total_tokens: 11, + models: {}, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 11, + apis: { + codex: { + total_requests: 1, + total_tokens: 11, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 11, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 6, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 11, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + expect(merged.usage?.total_requests).toBe(1); + expect(merged.usage?.total_tokens).toBe(11); + expect(merged.usage?.apis?.codex.total_requests).toBe(1); + expect(merged.usage?.apis?.codex.total_tokens).toBe(11); + expect(merged.usage?.apis?.codex.models?.['gpt-5.5'].total_requests).toBe(1); + expect(merged.usage?.apis?.codex.models?.['gpt-5.5'].total_tokens).toBe(0); + expect(merged.usage?.apis?.codex.models?.['gpt-5.5'].details).toHaveLength(1); + }); + + it('does not inflate model requests when enriching incomplete aggregate model details', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 2, + failure_count: 0, + total_tokens: 22, + apis: { + codex: { + total_requests: 2, + total_tokens: 22, + models: { + 'gpt-5.5': { + total_requests: 2, + total_tokens: 22, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 6, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 11, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 11, + apis: { + codex: { + total_requests: 1, + total_tokens: 11, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 11, + details: [ + { + timestamp: '2026-05-05T18:46:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-plus.json', + auth_index: 'codex-user@example.com-plus.json', + tokens: { + input_tokens: 7, + output_tokens: 4, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 11, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(merged.usage?.total_tokens).toBe(22); + expect(merged.usage?.apis?.codex.total_requests).toBe(2); + expect(merged.usage?.apis?.codex.total_tokens).toBe(22); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.total_tokens).toBe(22); + expect(modelBucket?.details).toHaveLength(2); + }); + + it('fills aggregate detail gaps for repeated requests from the same account', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 2, + failure_count: 0, + total_tokens: 24, + apis: { + codex: { + total_requests: 2, + total_tokens: 24, + models: { + 'gpt-5.5': { + total_requests: 2, + total_tokens: 24, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 6, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 11, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 13, + apis: { + codex: { + total_requests: 1, + total_tokens: 13, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 13, + details: [ + { + timestamp: '2026-05-05T18:46:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 8, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 13, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(merged.usage?.total_tokens).toBe(24); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.total_tokens).toBe(24); + expect(modelBucket?.details?.map((detail) => detail.timestamp)).toEqual([ + '2026-05-05T18:45:01.000Z', + '2026-05-05T18:46:01.000Z', + ]); + }); + + it('does not identity-dedupe tokenless logs before incomplete model detail gaps are filled', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 2, + failure_count: 0, + total_tokens: 23, + apis: { + codex: { + total_requests: 2, + total_tokens: 23, + models: { + 'gpt-5.5': { + total_requests: 2, + total_tokens: 23, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'user@example.com', + auth_index: 'codex-auth', + tokens: { + input_tokens: 12, + output_tokens: 8, + reasoning_tokens: 0, + cached_tokens: 3, + total_tokens: 23, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 0, + apis: { + codex: { + total_requests: 1, + total_tokens: 0, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 0, + details: [ + { + timestamp: '2026-05-05T18:45:03.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 0, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.details).toHaveLength(2); + }); + + it('dedupes tokenless local OAuth logs against delayed complete management details', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 23, + apis: { + codex: { + total_requests: 1, + total_tokens: 23, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 23, + details: [ + { + timestamp: '2026-05-05T18:47:30.000Z', + source: 'user@example.com', + auth_index: 'codex-auth', + request_id: 'req-1', + tokens: { + input_tokens: 12, + output_tokens: 8, + reasoning_tokens: 0, + cached_tokens: 3, + total_tokens: 23, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 0, + apis: { + codex: { + total_requests: 1, + total_tokens: 0, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 0, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + request_id: 'req-1', + tokens: { + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 0, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(1); + expect(merged.usage?.total_tokens).toBe(23); + expect(modelBucket?.total_requests).toBe(1); + expect(modelBucket?.details).toHaveLength(1); + expect(modelBucket?.details?.[0]?.tokens.total_tokens).toBe(23); + }); + + it('does not dedupe distinct Codex auth variants for the same email', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 23, + apis: { + codex: { + total_requests: 1, + total_tokens: 23, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 23, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'user@example.com', + auth_index: 'codex-pro-index', + request_id: 'req-pro', + tokens: { + input_tokens: 12, + output_tokens: 8, + reasoning_tokens: 0, + cached_tokens: 3, + total_tokens: 23, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 0, + apis: { + codex: { + total_requests: 1, + total_tokens: 0, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 0, + details: [ + { + timestamp: '2026-05-05T18:45:07.000Z', + source: 'provider=codex auth_file=codex-user@example.com-free.json', + auth_index: 'codex-user@example.com-free.json', + request_id: 'req-free', + tokens: { + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 0, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.details).toHaveLength(2); + }); + + it('dedupes duplicate logs before filling partial aggregate detail gaps', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 2, + failure_count: 0, + total_tokens: 23, + apis: { + codex: { + total_requests: 2, + total_tokens: 23, + models: { + 'gpt-5.5': { + total_requests: 2, + total_tokens: 23, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'user@example.com', + auth_index: 'codex-auth', + request_id: 'req-1', + tokens: { + input_tokens: 12, + output_tokens: 8, + reasoning_tokens: 0, + cached_tokens: 3, + total_tokens: 23, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 2, + failure_count: 0, + total_tokens: 0, + apis: { + codex: { + total_requests: 2, + total_tokens: 0, + models: { + 'gpt-5.5': { + total_requests: 2, + total_tokens: 0, + details: [ + { + timestamp: '2026-05-05T18:45:02.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + request_id: 'req-1', + tokens: { + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 0, + }, + failed: false, + }, + { + timestamp: '2026-05-05T18:45:10.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + request_id: 'req-2', + tokens: { + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 0, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.details?.map((detail) => detail.request_id)).toEqual(['req-1', 'req-2']); + }); + + it('keeps distinct complete requests from the same account and model', () => { + const merged = mergeUsageResponseWithMissingDetails( + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 11, + apis: { + codex: { + total_requests: 1, + total_tokens: 11, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 11, + details: [ + { + timestamp: '2026-05-05T18:45:01.000Z', + source: 'user@example.com', + auth_index: 'codex-auth', + tokens: { + input_tokens: 6, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 11, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + }, + { + failed_requests: 0, + usage: { + total_requests: 1, + success_count: 1, + failure_count: 0, + total_tokens: 13, + apis: { + codex: { + total_requests: 1, + total_tokens: 13, + models: { + 'gpt-5.5': { + total_requests: 1, + total_tokens: 13, + details: [ + { + timestamp: '2026-05-05T18:46:01.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + tokens: { + input_tokens: 8, + output_tokens: 5, + reasoning_tokens: 0, + cached_tokens: 0, + total_tokens: 13, + }, + failed: false, + }, + ], + }, + }, + }, + }, + }, + } + ); + + const modelBucket = merged.usage?.apis?.codex.models?.['gpt-5.5']; + expect(merged.usage?.total_requests).toBe(2); + expect(merged.usage?.total_tokens).toBe(24); + expect(modelBucket?.total_requests).toBe(2); + expect(modelBucket?.total_tokens).toBe(24); + expect(modelBucket?.details).toHaveLength(2); + }); +}); + +describe('fetchCliproxyStats', () => { + it('builds account stats from queue records and normalizes OAuth auth filenames', async () => { + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = requestUrl(input); + + if (url.endsWith('/v0/management/usage')) { + return jsonResponse({ error: 'not found' }, 404); + } + if (url.includes('/v0/management/usage-queue?count=1000')) { + return jsonResponse([createCodexQueueRecord()]); + } + if (url.endsWith('/v0/management/auth-files')) { + return jsonResponse({ files: [] }); + } + throw new Error(`unexpected URL: ${url}`); + }) as typeof fetch; + + const stats = await runWithScopedConfigDir(ccsDir, () => fetchCliproxyStats(19203)); + + expect(stats?.totalRequests).toBe(1); + expect(stats?.requestsByProvider).toEqual({ codex: 1 }); + expect(stats?.requestsByModel).toEqual({ 'gpt-5.5': 1 }); + expect(stats?.accountStats['codex:user@example.com']).toMatchObject({ + accountKey: 'codex:user@example.com', + provider: 'codex', + source: 'user@example.com', + successCount: 1, + failureCount: 0, + totalTokens: 23, + }); + }); +}); diff --git a/src/cliproxy/services/__tests__/stats-transformer.test.ts b/src/cliproxy/services/__tests__/stats-transformer.test.ts index 3627d56a..54d408b9 100644 --- a/src/cliproxy/services/__tests__/stats-transformer.test.ts +++ b/src/cliproxy/services/__tests__/stats-transformer.test.ts @@ -334,4 +334,211 @@ describe('buildCliproxyStatsFromUsageResponse', () => { failureCount: 1, }); }); + + it('normalizes OAuth auth filenames before building account stats keys', () => { + const usage: CliproxyUsageApiResponse = { + usage: { + total_requests: 1, + apis: { + codex: { + total_requests: 1, + models: { + 'gpt-5.5': { + total_requests: 1, + details: [ + createDetail({ + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + }), + ], + }, + }, + }, + }, + }, + }; + + const stats = buildCliproxyStatsFromUsageResponse(usage); + + expect(stats.accountStats['codex:user@example.com']).toMatchObject({ + provider: 'codex', + source: 'user@example.com', + successCount: 1, + failureCount: 0, + }); + }); + + it('normalizes OAuth-prefixed auth filename sources before building account stats keys', () => { + const usage: CliproxyUsageApiResponse = { + usage: { + total_requests: 2, + apis: { + codex: { + total_requests: 2, + models: { + 'gpt-5.5': { + total_requests: 2, + details: [ + createDetail({ + source: 'oauth|codex-user@example.com-pro.json', + auth_index: 'oauth|codex-user@example.com-pro.json', + }), + createDetail({ + source: 'provider=codex auth_file=oauth|codex-user@example.com-pro.json', + auth_index: 'oauth|codex-user@example.com-pro.json', + timestamp: '2025-03-26T10:01:00.000Z', + }), + ], + }, + }, + }, + }, + }, + }; + + const stats = buildCliproxyStatsFromUsageResponse(usage); + + expect(stats.accountStats['codex:user@example.com']).toMatchObject({ + provider: 'codex', + source: 'user@example.com', + successCount: 2, + failureCount: 0, + }); + expect(stats.accountStats['codex:oauth|codex-user@example.com']).toBeUndefined(); + }); + + it('uses detail-derived success and failure counts when aggregate counters are stale', () => { + const usage: CliproxyUsageApiResponse = { + failed_requests: 0, + usage: { + total_requests: 2, + success_count: 0, + failure_count: 0, + apis: { + codex: { + total_requests: 2, + models: { + 'gpt-5.5': { + total_requests: 2, + details: [ + createDetail({ + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + }), + createDetail({ + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + timestamp: '2025-03-26T10:01:00.000Z', + failed: true, + }), + ], + }, + }, + }, + }, + }, + }; + + const stats = buildCliproxyStatsFromUsageResponse(usage); + + expect(stats.successCount).toBe(1); + expect(stats.failureCount).toBe(1); + expect(stats.quotaExceededCount).toBe(1); + expect(stats.accountStats['codex:user@example.com']).toMatchObject({ + successCount: 1, + failureCount: 1, + }); + }); + + it('uses detail-derived totals and latest timestamps when aggregate request counts are stale', () => { + const usage: CliproxyUsageApiResponse = { + failed_requests: 0, + usage: { + total_requests: 0, + success_count: 0, + failure_count: 0, + apis: { + codex: { + total_requests: 0, + models: { + 'gpt-5.5': { + total_requests: 0, + details: [ + createDetail({ + timestamp: '2025-03-26T10:02:00.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + failed: true, + }), + createDetail({ + timestamp: '2025-03-26T10:01:00.000Z', + source: 'provider=codex auth_file=codex-user@example.com-pro.json', + auth_index: 'codex-user@example.com-pro.json', + }), + ], + }, + }, + }, + }, + }, + }; + + const stats = buildCliproxyStatsFromUsageResponse(usage); + + expect(stats.totalRequests).toBe(2); + expect(stats.requestsByModel['gpt-5.5']).toBe(2); + expect(stats.requestsByProvider.codex).toBe(2); + expect(stats.successCount).toBe(1); + expect(stats.failureCount).toBe(1); + expect(stats.accountStats['codex:user@example.com']).toMatchObject({ + successCount: 1, + failureCount: 1, + lastUsedAt: '2025-03-26T10:02:00.000Z', + }); + }); + + it('does not strip plan-like suffixes from raw account identifiers', () => { + const usage: CliproxyUsageApiResponse = { + usage: { + total_requests: 2, + apis: { + codex: { + total_requests: 2, + models: { + 'gpt-5.5': { + total_requests: 2, + details: [ + createDetail({ + source: 'provider=codex auth_file=codex-user-free@example.com.json', + auth_index: 'codex-user-free@example.com.json', + }), + createDetail({ + source: 'codex-user@example.com-pro', + auth_index: 'codex-user@example.com-pro', + timestamp: '2025-03-26T10:01:00.000Z', + }), + ], + }, + }, + }, + }, + }, + }; + + const stats = buildCliproxyStatsFromUsageResponse(usage); + + expect(stats.accountStats['codex:user-free@example.com']).toMatchObject({ + provider: 'codex', + source: 'user-free@example.com', + successCount: 1, + failureCount: 0, + }); + expect(stats.accountStats['codex:user@example.com-pro']).toMatchObject({ + provider: 'codex', + source: 'user@example.com-pro', + successCount: 1, + failureCount: 0, + }); + expect(stats.accountStats['codex:user@example.com']).toBeUndefined(); + }); }); diff --git a/src/cliproxy/services/oauth-usage-log-transformer.ts b/src/cliproxy/services/oauth-usage-log-transformer.ts new file mode 100644 index 00000000..a99f5e99 --- /dev/null +++ b/src/cliproxy/services/oauth-usage-log-transformer.ts @@ -0,0 +1,196 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { getCliproxyWritablePath } from '../config/path-resolver'; +import type { CliproxyUsageApiResponse } from './stats-fetcher'; +import { + buildUsageResponseFromQueueRecords, + hasUsageDetails, +} from './usage-compatibility-transformer'; + +interface PendingOAuthRequest { + timestamp?: string; + provider: string; + model: string; + authFile: string; + requestId?: string; +} + +interface ProviderCompletion { + timestamp?: string; + provider: string; + requestId?: string; + failed: boolean; +} + +function unquote(value: string): string { + return value.trim().replace(/^['"]|['"]$/g, ''); +} + +function extractTimestamp(line: string): string | undefined { + const isoTimestamp = line.match(/\d{4}-\d{2}-\d{2}T[^\s\]]+/)?.[0]; + if (isoTimestamp) { + return isoTimestamp.replace(/[,\]]+$/, ''); + } + + const bracketedTimestamp = line + .match(/^\[(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})\]/) + ?.slice(1, 3); + if (!bracketedTimestamp) { + return undefined; + } + + const [datePart, timePart] = bracketedTimestamp; + const [year, month, day] = datePart.split('-').map(Number); + const [hour, minute, second] = timePart.split(':').map(Number); + return new Date(year, month - 1, day, hour, minute, second).toISOString(); +} + +function extractRequestId(line: string): string | undefined { + const explicitRequestId = line.match( + /\b(?:request[_-]?id|req[_-]?id|rid)[=:]\s*([A-Za-z0-9._:-]+)/i + )?.[1]; + if (explicitRequestId) { + return explicitRequestId; + } + + const bracketedRequestId = line.match(/^\[[^\]]+\]\s+\[([^\]]+)\]/)?.[1]?.trim(); + return bracketedRequestId && bracketedRequestId !== '--------' ? bracketedRequestId : undefined; +} + +function parseOAuthSelection(line: string): PendingOAuthRequest | null { + const match = line.match( + /Use OAuth\s+provider=([^\s]+)\s+auth_file=("[^"]+"|'[^']+'|[^\s]+)\s+for model\s+("[^"]+"|'[^']+'|[^\s]+)/i + ); + if (!match) { + return null; + } + + return { + timestamp: extractTimestamp(line), + provider: unquote(match[1] ?? 'unknown'), + authFile: unquote(match[2] ?? 'unknown'), + model: unquote(match[3] ?? 'unknown'), + requestId: extractRequestId(line), + }; +} + +function parseProviderCompletion(line: string): ProviderCompletion | null { + const match = line.match(/\b(?:POST|GET)\s+"?\/api\/provider\/([^/"\s]+)\//i); + if (!match) { + return null; + } + + return { + timestamp: extractTimestamp(line), + provider: unquote(match[1] ?? 'unknown'), + requestId: extractRequestId(line), + failed: + /\b(?:failed|failure|error)\b/i.test(line) || + /\bstatus[=:]\s*[45]\d\d\b/i.test(line) || + /\s[45]\d\d(?:\s|$)/.test(line), + }; +} + +function addPendingRequest( + pending: PendingOAuthRequest, + byRequestId: Map, + byProvider: Map +): void { + if (pending.requestId) { + const previous = byRequestId.get(pending.requestId); + if (previous) { + const previousProviderQueue = byProvider.get(previous.provider) ?? []; + byProvider.set( + previous.provider, + previousProviderQueue.filter((entry) => entry !== previous) + ); + } + byRequestId.set(pending.requestId, pending); + } + + const providerQueue = byProvider.get(pending.provider) ?? []; + providerQueue.push(pending); + byProvider.set(pending.provider, providerQueue); +} + +function takePendingRequest( + completion: ProviderCompletion, + byRequestId: Map, + byProvider: Map +): PendingOAuthRequest | null { + const byId = completion.requestId ? byRequestId.get(completion.requestId) : undefined; + if (byId) { + byRequestId.delete(completion.requestId ?? ''); + const providerQueue = byProvider.get(byId.provider) ?? []; + byProvider.set( + byId.provider, + providerQueue.filter((entry) => entry !== byId) + ); + return byId; + } + + const providerQueue = byProvider.get(completion.provider) ?? []; + const next = providerQueue.shift(); + byProvider.set(completion.provider, providerQueue); + if (next?.requestId) { + byRequestId.delete(next.requestId); + } + return next ?? null; +} + +export function buildUsageResponseFromCliproxyLogLines(lines: string[]): CliproxyUsageApiResponse { + const pendingByRequestId = new Map(); + const pendingByProvider = new Map(); + const records: unknown[] = []; + + for (const line of lines) { + const selection = parseOAuthSelection(line); + if (selection) { + addPendingRequest(selection, pendingByRequestId, pendingByProvider); + continue; + } + + const completion = parseProviderCompletion(line); + if (!completion) { + continue; + } + + const pending = takePendingRequest(completion, pendingByRequestId, pendingByProvider); + if (!pending) { + continue; + } + + records.push({ + timestamp: completion.timestamp ?? pending.timestamp ?? '1970-01-01T00:00:00.000Z', + provider: pending.provider, + model: pending.model, + source: `provider=${pending.provider} auth_file=${pending.authFile}`, + auth_index: pending.authFile, + request_id: completion.requestId ?? pending.requestId, + failed: completion.failed, + tokens: {}, + }); + } + + return buildUsageResponseFromQueueRecords(records); +} + +function readLogFile(filePath: string): string | null { + if (!fs.existsSync(filePath)) { + return null; + } + + return fs.readFileSync(filePath, 'utf-8'); +} + +export function buildUsageResponseFromCliproxyMainLog(): CliproxyUsageApiResponse | null { + const logPath = path.join(getCliproxyWritablePath(), 'logs', 'main.log'); + const contents = readLogFile(logPath); + if (!contents) { + return null; + } + + const response = buildUsageResponseFromCliproxyLogLines(contents.split(/\r?\n/)); + return hasUsageDetails(response) ? response : null; +} diff --git a/src/cliproxy/services/stats-fetcher.ts b/src/cliproxy/services/stats-fetcher.ts index 7448dadd..272e8ffd 100644 --- a/src/cliproxy/services/stats-fetcher.ts +++ b/src/cliproxy/services/stats-fetcher.ts @@ -13,6 +13,22 @@ import { buildManagementHeaders, } from '../proxy/proxy-target-resolver'; import { buildCliproxyStatsFromUsageResponse } from './stats-transformer'; +import { buildUsageResponseFromCliproxyMainLog } from './oauth-usage-log-transformer'; +import { + buildUsageResponseFromApiKeyUsage, + buildUsageResponseFromQueueRecords, + hasUsageDetails, + hasUsageTotals, + mergeUsageResponseWithMissingDetails, + mergeUsageResponses, +} from './usage-compatibility-transformer'; + +interface ManagementJsonResult { + ok: boolean; + status: number; + data: T | null; + cacheKey: string; +} /** Per-account usage statistics */ export interface AccountUsageStats { @@ -65,6 +81,7 @@ export interface CliproxyRequestDetail { timestamp: string; source: string; auth_index: string | number; + request_id?: string; tokens: { input_tokens: number; output_tokens: number; @@ -112,6 +129,11 @@ export interface CliproxyManagementAuthFile { name?: string; } +const cachedUsageQueueResponses = new Map(); +const USAGE_QUEUE_BATCH_SIZE = 1000; +const USAGE_QUEUE_MAX_BATCHES = 100; +const USAGE_QUEUE_DRAIN_TIMEOUT_MS = 30_000; + /** * Fetch usage statistics from CLIProxyAPI management API * @param port CLIProxyAPI port (default: 8317) @@ -142,15 +164,159 @@ export async function fetchCliproxyStats(port?: number): Promise { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 5000); + let localLogUsage: CliproxyUsageApiResponse | null | undefined; + const getLocalLogUsage = (): CliproxyUsageApiResponse | null => { + if (localLogUsage !== undefined) { + return localLogUsage; + } + localLogUsage = getProxyTarget().isRemote ? null : buildUsageResponseFromCliproxyMainLog(); + return localLogUsage; + }; + + const mergeLocalLogUsage = (response: CliproxyUsageApiResponse): CliproxyUsageApiResponse => + mergeUsageResponseWithMissingDetails(response, getLocalLogUsage()); + const mergeAggregateWithDetails = ( + aggregate: CliproxyUsageApiResponse, + details: CliproxyUsageApiResponse + ): CliproxyUsageApiResponse => + mergeUsageResponseWithMissingDetails(aggregate, details, { appendExtraDetails: false }); + + let legacyAggregateUsage: CliproxyUsageApiResponse | null = null; + const legacyUsage = await fetchManagementJson( + '/v0/management/usage', + port + ); + if (legacyUsage?.ok && legacyUsage.data) { + if (hasUsageDetails(legacyUsage.data)) { + return mergeLocalLogUsage(legacyUsage.data); + } + legacyAggregateUsage = legacyUsage.data; + } + + let cachedUsageQueueResponse: CliproxyUsageApiResponse | undefined; + const usageQueue = await fetchUsageQueueRecords(port); + if (usageQueue?.cacheKey) { + cachedUsageQueueResponse = cachedUsageQueueResponses.get(usageQueue.cacheKey); + } + + if (usageQueue && Array.isArray(usageQueue.data)) { + const queueResponse = buildUsageResponseFromQueueRecords(usageQueue.data); + if (hasUsageDetails(queueResponse)) { + const mergedUsageQueueResponse = cachedUsageQueueResponse + ? mergeUsageResponses(cachedUsageQueueResponse, queueResponse) + : queueResponse; + cachedUsageQueueResponses.set(usageQueue.cacheKey, mergedUsageQueueResponse); + cachedUsageQueueResponse = mergedUsageQueueResponse; + if (usageQueue.ok) { + const response = legacyAggregateUsage + ? mergeAggregateWithDetails(legacyAggregateUsage, mergedUsageQueueResponse) + : mergedUsageQueueResponse; + return mergeLocalLogUsage(response); + } + } + } + + if (legacyAggregateUsage) { + const response = cachedUsageQueueResponse + ? mergeAggregateWithDetails(legacyAggregateUsage, cachedUsageQueueResponse) + : legacyAggregateUsage; + return mergeLocalLogUsage(response); + } + + const apiKeyUsage = await fetchManagementJson('/v0/management/api-key-usage', port); + if (apiKeyUsage?.ok && apiKeyUsage.data) { + const apiKeyResponseWithCachedDetails = cachedUsageQueueResponse + ? mergeAggregateWithDetails( + buildUsageResponseFromApiKeyUsage(apiKeyUsage.data), + cachedUsageQueueResponse + ) + : buildUsageResponseFromApiKeyUsage(apiKeyUsage.data); + const apiKeyResponse = mergeLocalLogUsage(apiKeyResponseWithCachedDetails); + if (hasUsageTotals(apiKeyResponse)) { + return apiKeyResponse; + } + } + + if (cachedUsageQueueResponse) { + return mergeLocalLogUsage(cachedUsageQueueResponse); + } + + const logUsage = getLocalLogUsage(); + if (logUsage && hasUsageDetails(logUsage)) { + return logUsage; + } + + if (usageQueue?.ok) { + return buildUsageResponseFromQueueRecords([]); + } + + return null; +} + +async function fetchUsageQueueRecords( + port?: number +): Promise | null> { + const records: unknown[] = []; + const seenFullBatchSignatures = new Set(); + let cacheKey = ''; + let status = 0; + const drainStartedAt = Date.now(); + + for (let batchCount = 0; batchCount < USAGE_QUEUE_MAX_BATCHES; batchCount++) { + const result = await fetchManagementJson( + `/v0/management/usage-queue?count=${USAGE_QUEUE_BATCH_SIZE}`, + port + ); + if (!result) { + return records.length > 0 ? { ok: false, status, data: records, cacheKey } : null; + } + + cacheKey ||= result.cacheKey; + status = result.status; + if (!result.ok || !Array.isArray(result.data)) { + return records.length > 0 ? { ok: false, status, data: records, cacheKey } : result; + } + + if (result.data.length === USAGE_QUEUE_BATCH_SIZE) { + const batchSignature = createUsageQueueBatchSignature(result.data); + if (seenFullBatchSignatures.has(batchSignature)) { + return { ok: false, status, data: records, cacheKey }; + } + seenFullBatchSignatures.add(batchSignature); + } + + records.push(...result.data); + if (result.data.length < USAGE_QUEUE_BATCH_SIZE) { + return { ok: true, status, data: records, cacheKey }; + } + + if (Date.now() - drainStartedAt >= USAGE_QUEUE_DRAIN_TIMEOUT_MS) { + return { ok: false, status, data: records, cacheKey }; + } + } + + return { ok: false, status, data: records, cacheKey }; +} + +function createUsageQueueBatchSignature(records: unknown[]): string { + return JSON.stringify(records); +} + +async function fetchManagementJson( + endpointPath: string, + port?: number, + timeoutMs = 5000 +): Promise | null> { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + + try { const target = getProxyTarget(); if (port !== undefined && !target.isRemote) { target.port = port; } - const url = buildProxyUrl(target, '/v0/management/usage'); + const url = buildProxyUrl(target, endpointPath); const headers = target.isRemote ? buildManagementHeaders(target) @@ -161,51 +327,46 @@ export async function fetchCliproxyUsageRaw( headers, }); - clearTimeout(timeoutId); - if (!response.ok) { - return null; + return { ok: false, status: response.status, data: null, cacheKey: url }; } - return (await response.json()) as CliproxyUsageApiResponse; + return { + ok: true, + status: response.status, + data: (await response.json()) as T, + cacheKey: url, + }; } catch { return null; + } finally { + clearTimeout(timeoutId); } } async function fetchCliproxyAuthFiles(port?: number): Promise { try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 5000); - - const target = getProxyTarget(); - if (port !== undefined && !target.isRemote) { - target.port = port; - } - const url = buildProxyUrl(target, '/v0/management/auth-files'); - - const headers = target.isRemote - ? buildManagementHeaders(target) - : { Accept: 'application/json', Authorization: `Bearer ${getEffectiveManagementSecret()}` }; - - const response = await fetch(url, { - signal: controller.signal, - headers, - }); - - clearTimeout(timeoutId); - - if (!response.ok) { + const result = await fetchManagementJson<{ files?: CliproxyManagementAuthFile[] }>( + '/v0/management/auth-files', + port + ); + if (!result?.ok || !result.data) { return null; } - const data = (await response.json()) as { files?: CliproxyManagementAuthFile[] }; + const data = result.data; return Array.isArray(data.files) ? data.files : null; } catch { return null; } } +export const __testExports = { + clearCachedUsageQueueResponse(): void { + cachedUsageQueueResponses.clear(); + }, +}; + /** OpenAI-compatible model object from /v1/models endpoint */ export interface CliproxyModel { id: string; diff --git a/src/cliproxy/services/stats-transformer.ts b/src/cliproxy/services/stats-transformer.ts index ed2be0ea..ca4bc7db 100644 --- a/src/cliproxy/services/stats-transformer.ts +++ b/src/cliproxy/services/stats-transformer.ts @@ -30,6 +30,44 @@ function normalizeProvider(provider: string): string { return mapExternalProviderName(normalized) ?? normalized; } +function extractAuthFilenameFromSource(source: string): string { + const authFileMatch = source.match(/(?:^|\s)auth_file=("[^"]+"|'[^']+'|[^\s]+)/i); + const rawValue = authFileMatch?.[1] ?? source; + const value = rawValue.trim().replace(/^['"]|['"]$/g, ''); + const filenameCandidate = value.split('|').pop() ?? value; + return filenameCandidate.split(/[\\/]/).pop() ?? filenameCandidate.trim(); +} + +function stripKnownAuthPlanSuffix(value: string): string { + return value.replace(/-(?:free|plus|pro|team|business|enterprise)$/i, ''); +} + +function normalizeAuthFilenameSource(provider: string, source: string): string | null { + const filename = extractAuthFilenameFromSource(source); + const normalizedProvider = normalizeProvider(provider); + const hadJsonExtension = /\.json$/i.test(filename); + let candidate = filename.replace(/\.json$/i, ''); + const providerPrefix = `${normalizedProvider}-`; + if (candidate.toLowerCase().startsWith(providerPrefix)) { + candidate = candidate.slice(providerPrefix.length); + } + + candidate = candidate.replace(/^[a-f0-9]{8}[-_]/i, ''); + if (hadJsonExtension) { + candidate = stripKnownAuthPlanSuffix(candidate); + } + + const parts = candidate.split('@'); + if (parts.length < 2) { + return null; + } + + const localPart = parts[0]?.trim(); + const domainPart = parts.slice(1).join('@').trim(); + const email = `${localPart}@${domainPart}`; + return /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(email) ? email.toLowerCase() : null; +} + function buildAuthIndexLookup( authFiles: CliproxyManagementAuthFile[] | undefined ): ReadonlyMap { @@ -106,12 +144,38 @@ function resolveSourceForDetail( const source = detail.source?.trim(); if (source) { - return source; + return normalizeAuthFilenameSource(resolvedProvider, source) ?? source; } return resolvedAuthFile?.source ?? 'unknown'; } +function resolveCountWithDetails(aggregateCount: number | undefined, detailCount: number): number { + if (aggregateCount === undefined) { + return detailCount; + } + + return aggregateCount < detailCount ? detailCount : aggregateCount; +} + +function shouldReplaceLastUsedAt(current: string | undefined, next: string | undefined): boolean { + if (!next) { + return false; + } + + const nextTime = Date.parse(next); + if (!Number.isFinite(nextTime)) { + return false; + } + + if (!current) { + return true; + } + + const currentTime = Date.parse(current); + return !Number.isFinite(currentTime) || nextTime > currentTime; +} + export function buildCliproxyStatsFromUsageResponse( data: CliproxyUsageApiResponse, options: BuildCliproxyStatsOptions = {} @@ -129,7 +193,7 @@ export function buildCliproxyStatsFromUsageResponse( if (usage?.apis) { for (const [provider, providerData] of Object.entries(usage.apis)) { - let sawProviderDetail = false; + let providerDetailCount = 0; if (!providerData.models) { const normalizedProvider = normalizeProvider(provider); requestsByProvider[normalizedProvider] = @@ -138,14 +202,16 @@ export function buildCliproxyStatsFromUsageResponse( } for (const [model, modelData] of Object.entries(providerData.models)) { - requestsByModel[model] = modelData.total_requests ?? 0; + let modelDetailCount = 0; if (!modelData.details) { + requestsByModel[model] = (requestsByModel[model] ?? 0) + (modelData.total_requests ?? 0); continue; } for (const detail of modelData.details) { sawAnyDetail = true; - sawProviderDetail = true; + providerDetailCount++; + modelDetailCount++; const resolvedProvider = resolveProviderForDetail(provider, detail, authIndexLookup); const source = resolveSourceForDetail(resolvedProvider, detail, authIndexLookup); const accountKey = buildQualifiedAccountStatsKey(resolvedProvider, source); @@ -172,26 +238,52 @@ export function buildCliproxyStatsFromUsageResponse( const tokens = detail.tokens?.total_tokens ?? 0; accountStats[accountKey].totalTokens += tokens; - accountStats[accountKey].lastUsedAt = detail.timestamp; + if (shouldReplaceLastUsedAt(accountStats[accountKey].lastUsedAt, detail.timestamp)) { + accountStats[accountKey].lastUsedAt = detail.timestamp; + } totalInputTokens += detail.tokens?.input_tokens ?? 0; totalOutputTokens += detail.tokens?.output_tokens ?? 0; } + + requestsByModel[model] = + (requestsByModel[model] ?? 0) + + resolveCountWithDetails(modelData.total_requests, modelDetailCount); } - if (!sawProviderDetail) { + const providerTotal = providerData.total_requests ?? 0; + if (providerDetailCount === 0) { const normalizedProvider = normalizeProvider(provider); requestsByProvider[normalizedProvider] = - (requestsByProvider[normalizedProvider] ?? 0) + (providerData.total_requests ?? 0); + (requestsByProvider[normalizedProvider] ?? 0) + providerTotal; + } else if (providerTotal > providerDetailCount) { + const normalizedProvider = normalizeProvider(provider); + requestsByProvider[normalizedProvider] = + (requestsByProvider[normalizedProvider] ?? 0) + (providerTotal - providerDetailCount); } } } + const aggregateFailureCount = usage?.failure_count ?? data.failed_requests; + const successCount = sawAnyDetail + ? resolveCountWithDetails(usage?.success_count, totalSuccessCount) + : (usage?.success_count ?? 0); + const failureCount = sawAnyDetail + ? resolveCountWithDetails(aggregateFailureCount, totalFailureCount) + : (aggregateFailureCount ?? 0); + const providerTotalRequests = Object.values(requestsByProvider).reduce( + (total, count) => total + count, + 0 + ); + const totalRequests = Math.max( + usage?.total_requests ?? 0, + successCount + failureCount, + providerTotalRequests + ); + return { - totalRequests: usage?.total_requests ?? 0, - successCount: sawAnyDetail ? totalSuccessCount : (usage?.success_count ?? 0), - failureCount: sawAnyDetail - ? totalFailureCount - : (usage?.failure_count ?? data.failed_requests ?? 0), + totalRequests, + successCount, + failureCount, tokens: { input: totalInputTokens, output: totalOutputTokens, @@ -200,7 +292,7 @@ export function buildCliproxyStatsFromUsageResponse( requestsByModel, requestsByProvider, accountStats, - quotaExceededCount: usage?.failure_count ?? data.failed_requests ?? 0, + quotaExceededCount: failureCount, retryCount: 0, collectedAt: new Date().toISOString(), }; diff --git a/src/cliproxy/services/usage-compatibility-transformer.ts b/src/cliproxy/services/usage-compatibility-transformer.ts new file mode 100644 index 00000000..dd2714cb --- /dev/null +++ b/src/cliproxy/services/usage-compatibility-transformer.ts @@ -0,0 +1,632 @@ +import type { CliproxyRequestDetail, CliproxyUsageApiResponse } from './stats-fetcher'; + +interface CliproxyUsageQueueRecord { + timestamp?: string; + provider?: string; + model?: string; + alias?: string; + source?: string; + auth_index?: string | number; + request_id?: string; + tokens?: Partial; + failed?: boolean; +} + +interface ApiKeyUsageEntry { + success?: number; + failed?: number; +} + +type ApiKeyUsageResponse = Record>; + +interface MergeMissingDetailsOptions { + appendExtraDetails?: boolean; +} + +function asRecord(value: unknown): Record | null { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null; +} + +function asString(value: unknown, fallback: string): string { + return typeof value === 'string' && value.trim() ? value.trim() : fallback; +} + +function asNumber(value: unknown): number { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + + if (typeof value === 'string' && value.trim()) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + } + + return 0; +} + +function asBoolean(value: unknown): boolean { + return value === true; +} + +function normalizeTokens(rawTokens: unknown): CliproxyRequestDetail['tokens'] { + const tokens = asRecord(rawTokens) ?? {}; + const input = asNumber(tokens.input_tokens); + const output = asNumber(tokens.output_tokens); + const reasoning = asNumber(tokens.reasoning_tokens); + const cached = asNumber(tokens.cached_tokens); + const explicitTotal = asNumber(tokens.total_tokens); + const total = explicitTotal || input + output + reasoning + cached; + + return { + input_tokens: input, + output_tokens: output, + reasoning_tokens: reasoning, + cached_tokens: cached, + total_tokens: total, + }; +} + +function normalizeQueueRecord(record: unknown): CliproxyUsageQueueRecord | null { + const raw = asRecord(record); + if (!raw) { + return null; + } + + const provider = asString(raw.provider, 'unknown'); + const model = asString(raw.model, asString(raw.alias, 'unknown')); + const source = asString(raw.source, 'unknown'); + const authIndex = + typeof raw.auth_index === 'string' || typeof raw.auth_index === 'number' + ? raw.auth_index + : source; + + return { + timestamp: asString(raw.timestamp, new Date().toISOString()), + provider, + model, + alias: asString(raw.alias, model), + source, + auth_index: authIndex, + request_id: asString(raw.request_id, ''), + tokens: normalizeTokens(raw.tokens), + failed: asBoolean(raw.failed), + }; +} + +function ensureProviderBucket( + response: CliproxyUsageApiResponse, + provider: string +): NonNullable['apis']>[string] { + const usage = (response.usage ??= { apis: {} }); + const apis = (usage.apis ??= {}); + return (apis[provider] ??= { total_requests: 0, total_tokens: 0, models: {} }); +} + +function ensureModelBucket( + providerBucket: NonNullable['apis']>[string], + model: string +): NonNullable[string] { + const models = (providerBucket.models ??= {}); + return (models[model] ??= { total_requests: 0, total_tokens: 0, details: [] }); +} + +function addDetail( + response: CliproxyUsageApiResponse, + provider: string, + model: string, + detail: CliproxyRequestDetail +): void { + const usage = (response.usage ??= { apis: {} }); + const providerBucket = ensureProviderBucket(response, provider); + const modelBucket = ensureModelBucket(providerBucket, model); + const totalTokens = detail.tokens?.total_tokens ?? 0; + + usage.total_requests = (usage.total_requests ?? 0) + 1; + usage.total_tokens = (usage.total_tokens ?? 0) + totalTokens; + if (detail.failed) { + usage.failure_count = (usage.failure_count ?? 0) + 1; + response.failed_requests = (response.failed_requests ?? 0) + 1; + } else { + usage.success_count = (usage.success_count ?? 0) + 1; + } + + providerBucket.total_requests = (providerBucket.total_requests ?? 0) + 1; + providerBucket.total_tokens = (providerBucket.total_tokens ?? 0) + totalTokens; + modelBucket.total_requests = (modelBucket.total_requests ?? 0) + 1; + modelBucket.total_tokens = (modelBucket.total_tokens ?? 0) + totalTokens; + (modelBucket.details ??= []).push(detail); +} + +function createDetailSignature( + provider: string, + model: string, + detail: CliproxyRequestDetail +): string { + return [ + provider, + model, + detail.request_id?.trim() ?? '', + detail.timestamp, + detail.source, + String(detail.auth_index), + detail.tokens?.input_tokens ?? 0, + detail.tokens?.output_tokens ?? 0, + detail.tokens?.reasoning_tokens ?? 0, + detail.tokens?.cached_tokens ?? 0, + detail.tokens?.total_tokens ?? 0, + detail.failed ? '1' : '0', + ].join('|'); +} + +function collectResponseDetails( + response: CliproxyUsageApiResponse +): Array<{ provider: string; model: string; detail: CliproxyRequestDetail }> { + const entries: Array<{ provider: string; model: string; detail: CliproxyRequestDetail }> = []; + for (const [provider, providerData] of Object.entries(response.usage?.apis ?? {})) { + for (const [model, modelData] of Object.entries(providerData.models ?? {})) { + for (const detail of modelData.details ?? []) { + entries.push({ provider, model, detail }); + } + } + } + return entries; +} + +export function buildUsageResponseFromQueueRecords(records: unknown[]): CliproxyUsageApiResponse { + const response: CliproxyUsageApiResponse = { + failed_requests: 0, + usage: { + total_requests: 0, + success_count: 0, + failure_count: 0, + total_tokens: 0, + apis: {}, + }, + }; + + for (const rawRecord of records) { + const record = normalizeQueueRecord(rawRecord); + if (!record) { + continue; + } + + addDetail(response, record.provider ?? 'unknown', record.model ?? 'unknown', { + timestamp: record.timestamp ?? new Date().toISOString(), + source: record.source ?? 'unknown', + auth_index: record.auth_index ?? record.source ?? 'unknown', + request_id: record.request_id || undefined, + tokens: normalizeTokens(record.tokens), + failed: record.failed === true, + }); + } + + return response; +} + +export function mergeUsageResponses( + base: CliproxyUsageApiResponse, + incoming: CliproxyUsageApiResponse +): CliproxyUsageApiResponse { + const merged = buildUsageResponseFromQueueRecords([]); + const seen = new Set(); + + for (const entry of [...collectResponseDetails(base), ...collectResponseDetails(incoming)]) { + const signature = createDetailSignature(entry.provider, entry.model, entry.detail); + if (seen.has(signature)) { + continue; + } + seen.add(signature); + addDetail(merged, entry.provider, entry.model, entry.detail); + } + + return merged; +} + +function cloneUsageResponse(response: CliproxyUsageApiResponse): CliproxyUsageApiResponse { + return { + failed_requests: response.failed_requests ?? 0, + usage: { + total_requests: response.usage?.total_requests ?? 0, + success_count: response.usage?.success_count ?? 0, + failure_count: response.usage?.failure_count ?? 0, + total_tokens: response.usage?.total_tokens ?? 0, + apis: Object.fromEntries( + Object.entries(response.usage?.apis ?? {}).map(([provider, providerData]) => [ + provider, + { + total_requests: providerData.total_requests ?? 0, + total_tokens: providerData.total_tokens ?? 0, + models: Object.fromEntries( + Object.entries(providerData.models ?? {}).map(([model, modelData]) => [ + model, + { + total_requests: modelData.total_requests ?? 0, + total_tokens: modelData.total_tokens ?? 0, + details: (modelData.details ?? []).map((detail) => ({ + ...detail, + tokens: { ...detail.tokens }, + })), + }, + ]) + ), + }, + ]) + ), + }, + }; +} + +function cloneDetail(detail: CliproxyRequestDetail): CliproxyRequestDetail { + return { + ...detail, + tokens: { ...detail.tokens }, + }; +} + +function createMissingDetailMergeKey( + provider: string, + model: string, + detail: CliproxyRequestDetail +): string { + return [ + provider, + model, + detail.request_id?.trim() ?? '', + detail.timestamp, + detail.source?.trim() ?? '', + String(detail.auth_index ?? '').trim(), + detail.tokens?.input_tokens ?? 0, + detail.tokens?.output_tokens ?? 0, + detail.tokens?.reasoning_tokens ?? 0, + detail.tokens?.cached_tokens ?? 0, + detail.tokens?.total_tokens ?? 0, + detail.failed ? '1' : '0', + ].join('|'); +} + +function extractDuplicateIdentityValue(value: string): string { + const authFileMatch = value.match(/(?:^|\s)auth_file=("[^"]+"|'[^']+'|[^\s]+)/i); + const rawValue = authFileMatch?.[1] ?? value; + const unquotedValue = rawValue.trim().replace(/^['"]|['"]$/g, ''); + const pipeCandidate = unquotedValue.split('|').pop() ?? unquotedValue; + return pipeCandidate.split(/[\\/]/).pop() ?? pipeCandidate.trim(); +} + +function stripKnownAuthPlanSuffix(value: string): string { + return value.replace(/-(?:free|plus|pro|team|business|enterprise)$/i, ''); +} + +function normalizeDuplicateIdentity(provider: string, value: string | number | undefined): string { + const rawValue = String(value ?? '').trim(); + if (!rawValue) { + return ''; + } + + const normalizedProvider = provider.trim().toLowerCase(); + const identityValue = extractDuplicateIdentityValue(rawValue); + const hadJsonExtension = /\.json$/i.test(identityValue); + let candidate = identityValue.replace(/\.json$/i, ''); + const providerPrefix = `${normalizedProvider}-`; + if (candidate.toLowerCase().startsWith(providerPrefix)) { + candidate = candidate.slice(providerPrefix.length); + } + + candidate = candidate.replace(/^[a-f0-9]{8}[-_]/i, '').trim(); + if (hadJsonExtension) { + candidate = stripKnownAuthPlanSuffix(candidate); + } + + return candidate ? candidate.toLowerCase() : rawValue.toLowerCase(); +} + +function resolveCompleteDetailDuplicateIdentity( + provider: string, + detail: CliproxyRequestDetail +): string { + return ( + normalizeDuplicateIdentity(provider, detail.source) || + normalizeDuplicateIdentity(provider, detail.auth_index) || + 'unknown' + ); +} + +function createLikelyLogDuplicateKey( + provider: string, + model: string, + detail: CliproxyRequestDetail +): string { + return [ + provider, + model, + resolveCompleteDetailDuplicateIdentity(provider, detail), + detail.failed ? '1' : '0', + ].join('|'); +} + +function createRequestIdDuplicateKey( + provider: string, + model: string, + detail: CliproxyRequestDetail +): string { + const requestId = detail.request_id?.trim(); + return requestId ? [provider, model, requestId, detail.failed ? '1' : '0'].join('|') : ''; +} + +function parseDetailTimestamp(detail: CliproxyRequestDetail): number | null { + const timestampMs = Date.parse(detail.timestamp); + return Number.isFinite(timestampMs) ? timestampMs : null; +} + +const TOKENLESS_LOG_DUPLICATE_WINDOW_MS = 5_000; + +interface LikelyLogDuplicateCandidates { + byRequestId: Map; + byIdentity: Map; +} + +function detailHasTokenUsage(detail: CliproxyRequestDetail): boolean { + return ( + (detail.tokens?.input_tokens ?? 0) > 0 || + (detail.tokens?.output_tokens ?? 0) > 0 || + (detail.tokens?.reasoning_tokens ?? 0) > 0 || + (detail.tokens?.cached_tokens ?? 0) > 0 || + (detail.tokens?.total_tokens ?? 0) > 0 + ); +} + +function collectDetailMergeKeyCounts(response: CliproxyUsageApiResponse): Map { + const counts = new Map(); + for (const entry of collectResponseDetails(response)) { + const key = createMissingDetailMergeKey(entry.provider, entry.model, entry.detail); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +function collectLikelyLogDuplicateCandidates( + response: CliproxyUsageApiResponse +): LikelyLogDuplicateCandidates { + const candidates: LikelyLogDuplicateCandidates = { + byRequestId: new Map(), + byIdentity: new Map(), + }; + for (const [provider, providerData] of Object.entries(response.usage?.apis ?? {})) { + for (const [model, modelData] of Object.entries(providerData.models ?? {})) { + const details = modelData.details ?? []; + const canUseIdentityWindow = (modelData.total_requests ?? 0) <= details.length; + for (const detail of details) { + const requestIdKey = createRequestIdDuplicateKey(provider, model, detail); + if (requestIdKey) { + candidates.byRequestId.set( + requestIdKey, + (candidates.byRequestId.get(requestIdKey) ?? 0) + 1 + ); + } + + if (!canUseIdentityWindow) { + continue; + } + + const timestampMs = parseDetailTimestamp(detail); + if (timestampMs === null) { + continue; + } + + const key = createLikelyLogDuplicateKey(provider, model, detail); + const timestamps = candidates.byIdentity.get(key) ?? []; + timestamps.push(timestampMs); + candidates.byIdentity.set(key, timestamps); + } + } + } + return candidates; +} + +function consumeDetailKey(counts: Map, key: string): boolean { + const count = counts.get(key) ?? 0; + if (count <= 0) { + return false; + } + + if (count === 1) { + counts.delete(key); + } else { + counts.set(key, count - 1); + } + return true; +} + +function consumeCount(counts: Map, key: string): boolean { + const count = counts.get(key) ?? 0; + if (count <= 0) { + return false; + } + + if (count === 1) { + counts.delete(key); + } else { + counts.set(key, count - 1); + } + return true; +} + +function consumeLikelyLogDuplicateCandidate( + candidates: LikelyLogDuplicateCandidates, + provider: string, + model: string, + detail: CliproxyRequestDetail +): boolean { + if (detailHasTokenUsage(detail)) { + return false; + } + + const requestIdKey = createRequestIdDuplicateKey(provider, model, detail); + if (requestIdKey && consumeCount(candidates.byRequestId, requestIdKey)) { + return true; + } + + const timestampMs = parseDetailTimestamp(detail); + if (timestampMs === null) { + return false; + } + + const key = createLikelyLogDuplicateKey(provider, model, detail); + const timestamps = candidates.byIdentity.get(key) ?? []; + const index = timestamps.findIndex( + (candidateTimestampMs) => + Math.abs(candidateTimestampMs - timestampMs) <= TOKENLESS_LOG_DUPLICATE_WINDOW_MS + ); + if (index === -1) { + return false; + } + + timestamps.splice(index, 1); + if (timestamps.length === 0) { + candidates.byIdentity.delete(key); + } else { + candidates.byIdentity.set(key, timestamps); + } + return true; +} + +function countProviderDetails( + providerBucket: NonNullable['apis']>[string] +): number { + return Object.values(providerBucket.models ?? {}).reduce( + (total, modelData) => total + (modelData.details ?? []).length, + 0 + ); +} + +function appendDetailToExistingProvider( + merged: CliproxyUsageApiResponse, + provider: string, + model: string, + detail: CliproxyRequestDetail, + options: Required +): void { + const providerBucket = ensureProviderBucket(merged, provider); + const shouldFillAggregateOnly = + (providerBucket.total_requests ?? 0) > countProviderDetails(providerBucket); + if (!shouldFillAggregateOnly) { + if (!options.appendExtraDetails) { + return; + } + addDetail(merged, provider, model, cloneDetail(detail)); + return; + } + + const modelBucket = ensureModelBucket(providerBucket, model); + const shouldFillModelAggregateOnly = + (modelBucket.total_requests ?? 0) > (modelBucket.details ?? []).length; + + if (!shouldFillModelAggregateOnly) { + modelBucket.total_requests = (modelBucket.total_requests ?? 0) + 1; + } + (modelBucket.details ??= []).push(cloneDetail(detail)); +} + +export function mergeUsageResponseWithMissingDetails( + base: CliproxyUsageApiResponse, + incoming: CliproxyUsageApiResponse | null | undefined, + options: MergeMissingDetailsOptions = {} +): CliproxyUsageApiResponse { + if (!incoming || !hasUsageDetails(incoming)) { + return base; + } + + const normalizedOptions: Required = { + appendExtraDetails: options.appendExtraDetails ?? true, + }; + const merged = cloneUsageResponse(base); + const existingDetailCounts = collectDetailMergeKeyCounts(base); + const likelyLogDuplicateCandidates = collectLikelyLogDuplicateCandidates(base); + for (const entry of collectResponseDetails(incoming)) { + const mergeKey = createMissingDetailMergeKey(entry.provider, entry.model, entry.detail); + if (consumeDetailKey(existingDetailCounts, mergeKey)) { + continue; + } + + if ( + consumeLikelyLogDuplicateCandidate( + likelyLogDuplicateCandidates, + entry.provider, + entry.model, + entry.detail + ) + ) { + continue; + } + + if (base.usage?.apis?.[entry.provider]) { + appendDetailToExistingProvider( + merged, + entry.provider, + entry.model, + entry.detail, + normalizedOptions + ); + continue; + } + + addDetail(merged, entry.provider, entry.model, cloneDetail(entry.detail)); + } + + return merged; +} + +export function buildUsageResponseFromApiKeyUsage(rawResponse: unknown): CliproxyUsageApiResponse { + const response = buildUsageResponseFromQueueRecords([]); + const usage = response.usage ?? { + total_requests: 0, + success_count: 0, + failure_count: 0, + total_tokens: 0, + apis: {}, + }; + response.usage = usage; + const byProvider = asRecord(rawResponse) as ApiKeyUsageResponse | null; + if (!byProvider) { + return response; + } + + for (const [provider, sources] of Object.entries(byProvider)) { + const sourceRecords = asRecord(sources); + if (!sourceRecords) { + continue; + } + + let providerRequests = 0; + for (const entry of Object.values(sourceRecords)) { + const usageEntry = asRecord(entry); + if (!usageEntry) { + continue; + } + + const success = asNumber(usageEntry.success); + const failed = asNumber(usageEntry.failed); + providerRequests += success + failed; + usage.success_count = (usage.success_count ?? 0) + success; + usage.failure_count = (usage.failure_count ?? 0) + failed; + response.failed_requests = (response.failed_requests ?? 0) + failed; + } + + if (providerRequests > 0) { + const providerBucket = ensureProviderBucket(response, provider); + providerBucket.total_requests = (providerBucket.total_requests ?? 0) + providerRequests; + usage.total_requests = (usage.total_requests ?? 0) + providerRequests; + } + } + + return response; +} + +export function hasUsageDetails(response: CliproxyUsageApiResponse): boolean { + return collectResponseDetails(response).length > 0; +} + +export function hasUsageTotals(response: CliproxyUsageApiResponse): boolean { + return (response.usage?.total_requests ?? 0) > 0; +}