mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-16 14:16:43 +00:00
fix: normalize cliproxy v3 usage snapshots
This commit is contained in:
@@ -483,9 +483,8 @@ export async function reconcileExhaustedRotationAccounts(
|
||||
return [];
|
||||
}
|
||||
|
||||
const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import(
|
||||
'../accounts/account-safety'
|
||||
);
|
||||
const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } =
|
||||
await import('../accounts/account-safety');
|
||||
restoreExpiredQuotaPauses();
|
||||
|
||||
const config = loadOrCreateUnifiedConfig();
|
||||
@@ -593,9 +592,8 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
|
||||
return { proceed: true, accountId: defaultAccount?.id || '' };
|
||||
}
|
||||
|
||||
const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import(
|
||||
'../accounts/account-safety'
|
||||
);
|
||||
const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } =
|
||||
await import('../accounts/account-safety');
|
||||
restoreExpiredQuotaPauses();
|
||||
|
||||
const config = loadOrCreateUnifiedConfig();
|
||||
|
||||
@@ -112,9 +112,8 @@ export async function runImageAnalysisCheck(results: HealthCheck): Promise<void>
|
||||
* Fix image analysis configuration issues
|
||||
*/
|
||||
export async function fixImageAnalysisConfig(): Promise<boolean> {
|
||||
const { updateConfig, loadOrCreateUnifiedConfig } = await import(
|
||||
'../../config/config-loader-facade'
|
||||
);
|
||||
const { updateConfig, loadOrCreateUnifiedConfig } =
|
||||
await import('../../config/config-loader-facade');
|
||||
|
||||
const config = loadOrCreateUnifiedConfig();
|
||||
let fixed = false;
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
buildCliproxyUsageHistoryAggregates,
|
||||
extractCliproxyUsageHistoryDetails,
|
||||
mergeCliproxyUsageHistoryDetails,
|
||||
normalizeCliproxyUsageHistoryDetail,
|
||||
pruneCliproxyUsageHistoryDetails,
|
||||
type CliproxyUsageHistoryDetail,
|
||||
} from './cliproxy-usage-transformer';
|
||||
@@ -197,14 +198,27 @@ function readSnapshot(emitWarnings = true): CliproxyUsageSnapshot | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Array.isArray((snapshot as CliproxyUsageSnapshot).details)) {
|
||||
const details = (snapshot as CliproxyUsageSnapshot).details;
|
||||
if (!Array.isArray(details)) {
|
||||
if (emitWarnings) {
|
||||
console.log(info('CLIProxy snapshot details missing, will refresh on next sync'));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return snapshot as CliproxyUsageSnapshot;
|
||||
const normalizedDetails = details
|
||||
.map((detail) => normalizeCliproxyUsageHistoryDetail(detail))
|
||||
.filter((detail): detail is CliproxyUsageHistoryDetail => detail !== null);
|
||||
const { daily, hourly, monthly } = buildCliproxyUsageHistoryAggregates(normalizedDetails);
|
||||
|
||||
return {
|
||||
version: SNAPSHOT_VERSION,
|
||||
timestamp: Number(snapshot.timestamp),
|
||||
details: normalizedDetails,
|
||||
daily,
|
||||
hourly,
|
||||
monthly,
|
||||
};
|
||||
}
|
||||
|
||||
if (snapshot.version === 1 || snapshot.version === 2) {
|
||||
|
||||
@@ -64,30 +64,112 @@ function createHistoryDetail(
|
||||
detail: CliproxyRequestDetail
|
||||
): CliproxyUsageHistoryDetail {
|
||||
const pricingProvider = normalizeUsageProvider(provider) ?? provider.trim().toLowerCase();
|
||||
const inputTokens = detail.tokens?.input_tokens ?? 0;
|
||||
const outputTokens = detail.tokens?.output_tokens ?? 0;
|
||||
const cacheReadTokens = detail.tokens?.cached_tokens ?? 0;
|
||||
|
||||
return {
|
||||
model,
|
||||
provider: pricingProvider,
|
||||
timestamp: detail.timestamp,
|
||||
source: detail.source,
|
||||
authIndex: String(detail.auth_index),
|
||||
inputTokens: detail.tokens?.input_tokens ?? 0,
|
||||
outputTokens: detail.tokens?.output_tokens ?? 0,
|
||||
cacheReadTokens: detail.tokens?.cached_tokens ?? 0,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheReadTokens,
|
||||
requestCount: 1,
|
||||
cost: calculateCost(
|
||||
{
|
||||
inputTokens: detail.tokens?.input_tokens ?? 0,
|
||||
outputTokens: detail.tokens?.output_tokens ?? 0,
|
||||
cacheCreationTokens: 0,
|
||||
cacheReadTokens: detail.tokens?.cached_tokens ?? 0,
|
||||
},
|
||||
cost: calculateHistoryDetailCost(
|
||||
model,
|
||||
{ provider: pricingProvider }
|
||||
pricingProvider,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheReadTokens
|
||||
),
|
||||
failed: detail.failed,
|
||||
};
|
||||
}
|
||||
|
||||
function calculateHistoryDetailCost(
|
||||
model: string,
|
||||
provider: string | undefined,
|
||||
inputTokens: number,
|
||||
outputTokens: number,
|
||||
cacheReadTokens: number
|
||||
): number {
|
||||
return calculateCost(
|
||||
{
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationTokens: 0,
|
||||
cacheReadTokens,
|
||||
},
|
||||
model,
|
||||
provider ? { provider } : undefined
|
||||
);
|
||||
}
|
||||
|
||||
function normalizePersistedNumber(value: unknown, fallback = 0): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
|
||||
}
|
||||
|
||||
function normalizePersistedProvider(value: unknown): string | undefined {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
|
||||
const trimmed = value.trim();
|
||||
if (trimmed.length === 0) return undefined;
|
||||
|
||||
return normalizeUsageProvider(trimmed) ?? trimmed.toLowerCase();
|
||||
}
|
||||
|
||||
export function normalizeCliproxyUsageHistoryDetail(
|
||||
detail: unknown
|
||||
): CliproxyUsageHistoryDetail | null {
|
||||
if (!detail || typeof detail !== 'object') return null;
|
||||
|
||||
const candidate = detail as Record<string, unknown>;
|
||||
if (
|
||||
typeof candidate.model !== 'string' ||
|
||||
typeof candidate.timestamp !== 'string' ||
|
||||
typeof candidate.source !== 'string' ||
|
||||
!Number.isFinite(Date.parse(candidate.timestamp))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const provider = normalizePersistedProvider(candidate.provider);
|
||||
const inputTokens = normalizePersistedNumber(candidate.inputTokens);
|
||||
const outputTokens = normalizePersistedNumber(candidate.outputTokens);
|
||||
const cacheReadTokens = normalizePersistedNumber(candidate.cacheReadTokens);
|
||||
const requestCount = Math.max(1, normalizePersistedNumber(candidate.requestCount, 1));
|
||||
const cost = normalizePersistedNumber(
|
||||
candidate.cost,
|
||||
calculateHistoryDetailCost(
|
||||
candidate.model,
|
||||
provider,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheReadTokens
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
model: candidate.model,
|
||||
...(provider && { provider }),
|
||||
timestamp: candidate.timestamp,
|
||||
source: candidate.source,
|
||||
authIndex:
|
||||
typeof candidate.authIndex === 'string'
|
||||
? candidate.authIndex
|
||||
: String(candidate.authIndex ?? ''),
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheReadTokens,
|
||||
requestCount,
|
||||
cost,
|
||||
failed: candidate.failed === true,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// FLATTEN
|
||||
// ============================================================================
|
||||
@@ -143,12 +225,53 @@ function createHistorySignature(detail: CliproxyUsageHistoryDetail): string {
|
||||
].join('|');
|
||||
}
|
||||
|
||||
function createProviderlessHistorySignature(detail: CliproxyUsageHistoryDetail): string {
|
||||
return [
|
||||
detail.model,
|
||||
detail.timestamp,
|
||||
detail.source,
|
||||
detail.authIndex,
|
||||
detail.inputTokens,
|
||||
detail.outputTokens,
|
||||
detail.cacheReadTokens,
|
||||
detail.requestCount,
|
||||
detail.failed ? '1' : '0',
|
||||
].join('|');
|
||||
}
|
||||
|
||||
function hydrateProviderlessHistoryDetails(
|
||||
existing: CliproxyUsageHistoryDetail[],
|
||||
incoming: CliproxyUsageHistoryDetail[]
|
||||
): CliproxyUsageHistoryDetail[] {
|
||||
const incomingByProviderlessSignature = new Map<string, CliproxyUsageHistoryDetail[]>();
|
||||
for (const detail of incoming) {
|
||||
if (!detail.provider) continue;
|
||||
|
||||
const signature = createProviderlessHistorySignature(detail);
|
||||
incomingByProviderlessSignature.set(signature, [
|
||||
...(incomingByProviderlessSignature.get(signature) ?? []),
|
||||
detail,
|
||||
]);
|
||||
}
|
||||
|
||||
return existing.map((detail) => {
|
||||
if (detail.provider) return detail;
|
||||
|
||||
const matches = incomingByProviderlessSignature.get(createProviderlessHistorySignature(detail));
|
||||
const providers = new Set(matches?.map((match) => match.provider).filter(Boolean));
|
||||
if (!matches || providers.size !== 1) return detail;
|
||||
|
||||
return { ...detail, provider: matches[0].provider, cost: matches[0].cost };
|
||||
});
|
||||
}
|
||||
|
||||
export function mergeCliproxyUsageHistoryDetails(
|
||||
existing: CliproxyUsageHistoryDetail[],
|
||||
incoming: CliproxyUsageHistoryDetail[]
|
||||
): CliproxyUsageHistoryDetail[] {
|
||||
const hydratedExisting = hydrateProviderlessHistoryDetails(existing, incoming);
|
||||
const existingCounts = new Map<string, { detail: CliproxyUsageHistoryDetail; count: number }>();
|
||||
for (const detail of existing) {
|
||||
for (const detail of hydratedExisting) {
|
||||
const signature = createHistorySignature(detail);
|
||||
const entry = existingCounts.get(signature);
|
||||
if (entry) {
|
||||
|
||||
@@ -20,7 +20,10 @@ function fetchRawResponse(): Promise<CliproxyUsageApiResponse | null> {
|
||||
return Promise.resolve(rawResponse);
|
||||
}
|
||||
|
||||
function buildResponse(inputTokens: number, timestamp = '2026-03-02T12:00:00.000Z'): CliproxyUsageApiResponse {
|
||||
function buildResponse(
|
||||
inputTokens: number,
|
||||
timestamp = '2026-03-02T12:00:00.000Z'
|
||||
): CliproxyUsageApiResponse {
|
||||
return {
|
||||
usage: {
|
||||
apis: {
|
||||
@@ -175,6 +178,69 @@ describe('cliproxy usage syncer', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes old v3 snapshot details before loading and merging history', async () => {
|
||||
await runWithScopedConfigDir(ccsDir, async () => {
|
||||
const snapshotPath = path.join(ccsDir, 'cache', 'cliproxy-usage', 'latest.json');
|
||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
snapshotPath,
|
||||
JSON.stringify({
|
||||
version: 3,
|
||||
timestamp: Date.now() - 60_000,
|
||||
details: [
|
||||
{
|
||||
model: 'gemini-2.5-pro',
|
||||
timestamp: '2026-03-02T12:00:00.000Z',
|
||||
source: 'account-a',
|
||||
authIndex: '0',
|
||||
inputTokens: 100,
|
||||
outputTokens: 20,
|
||||
cacheReadTokens: 10,
|
||||
failed: false,
|
||||
},
|
||||
],
|
||||
daily: [
|
||||
{
|
||||
date: '2026-03-02',
|
||||
source: 'cliproxy',
|
||||
inputTokens: 100,
|
||||
outputTokens: 20,
|
||||
cacheCreationTokens: 0,
|
||||
cacheReadTokens: 10,
|
||||
cost: null,
|
||||
totalCost: null,
|
||||
modelsUsed: ['gemini-2.5-pro'],
|
||||
modelBreakdowns: [],
|
||||
},
|
||||
],
|
||||
hourly: [],
|
||||
monthly: [],
|
||||
}),
|
||||
'utf-8'
|
||||
);
|
||||
|
||||
const loaded = await loadCachedCliproxyData();
|
||||
expect(loaded.daily[0].inputTokens).toBe(100);
|
||||
expect(Number.isFinite(loaded.daily[0].totalCost)).toBe(true);
|
||||
expect(loaded.hourly[0].requestCount).toBe(1);
|
||||
|
||||
await syncCliproxyUsage(fetchRawResponse);
|
||||
|
||||
const cached = await loadCachedCliproxyData();
|
||||
expect(cached.daily).toHaveLength(1);
|
||||
expect(cached.daily[0].inputTokens).toBe(100);
|
||||
expect(cached.hourly[0].requestCount).toBe(1);
|
||||
|
||||
const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf-8')) as {
|
||||
details: Array<{ requestCount?: number; cost?: number; provider?: string }>;
|
||||
};
|
||||
expect(snapshot.details).toHaveLength(1);
|
||||
expect(snapshot.details[0].requestCount).toBe(1);
|
||||
expect(Number.isFinite(snapshot.details[0].cost)).toBe(true);
|
||||
expect(snapshot.details[0].provider).toBe('google');
|
||||
});
|
||||
});
|
||||
|
||||
it('migrates legacy v1 and v2 snapshots forward before merging new history', async () => {
|
||||
for (const version of [1, 2]) {
|
||||
await runWithScopedConfigDir(ccsDir, async () => {
|
||||
@@ -264,7 +330,9 @@ describe('cliproxy usage syncer', () => {
|
||||
|
||||
const cached = await loadCachedCliproxyData();
|
||||
expect(cached.daily.map((entry) => entry.date)).toEqual(['2026-03-02', '2026-03-01']);
|
||||
expect(cached.hourly.find((entry) => entry.hour === '2026-03-01 12:00')?.requestCount).toBe(7);
|
||||
expect(cached.hourly.find((entry) => entry.hour === '2026-03-01 12:00')?.requestCount).toBe(
|
||||
7
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -274,9 +342,7 @@ describe('cliproxy usage syncer', () => {
|
||||
await syncCliproxyUsage(() =>
|
||||
Promise.resolve(buildResponse(100, '2024-01-01T12:00:00.000Z'))
|
||||
);
|
||||
await syncCliproxyUsage(() =>
|
||||
Promise.resolve(buildResponse(200, new Date().toISOString()))
|
||||
);
|
||||
await syncCliproxyUsage(() => Promise.resolve(buildResponse(200, new Date().toISOString())));
|
||||
|
||||
const cached = await loadCachedCliproxyData();
|
||||
expect(cached.daily.some((entry) => entry.date === '2024-01-01')).toBe(false);
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
buildCliproxyUsageHistoryAggregates,
|
||||
extractCliproxyUsageHistoryDetails,
|
||||
mergeCliproxyUsageHistoryDetails,
|
||||
normalizeCliproxyUsageHistoryDetail,
|
||||
transformCliproxyToDailyUsage,
|
||||
transformCliproxyToHourlyUsage,
|
||||
transformCliproxyToMonthlyUsage,
|
||||
@@ -111,18 +112,12 @@ describe('cliproxy usage transformer', () => {
|
||||
expect(flat[0].provider).toBe('google');
|
||||
expect(
|
||||
flat.some(
|
||||
(entry) =>
|
||||
entry.failed === true &&
|
||||
entry.inputTokens === 40 &&
|
||||
entry.outputTokens === 10
|
||||
(entry) => entry.failed === true && entry.inputTokens === 40 && entry.outputTokens === 10
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
flat.some(
|
||||
(entry) =>
|
||||
entry.failed === true &&
|
||||
entry.inputTokens === 0 &&
|
||||
entry.outputTokens === 0
|
||||
(entry) => entry.failed === true && entry.inputTokens === 0 && entry.outputTokens === 0
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
@@ -134,6 +129,54 @@ describe('cliproxy usage transformer', () => {
|
||||
expect(merged).toHaveLength(details.length);
|
||||
});
|
||||
|
||||
it('normalizes old v3 history details before merging with provider-aware entries', () => {
|
||||
const [incoming] = extractCliproxyUsageHistoryDetails({
|
||||
usage: {
|
||||
apis: {
|
||||
gemini: {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
details: [
|
||||
{
|
||||
timestamp: '2026-03-01T10:15:00.000Z',
|
||||
source: 'account-a',
|
||||
auth_index: 0,
|
||||
tokens: {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 20,
|
||||
total_tokens: 170,
|
||||
},
|
||||
failed: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const legacyDetail = normalizeCliproxyUsageHistoryDetail({
|
||||
model: incoming.model,
|
||||
timestamp: incoming.timestamp,
|
||||
source: incoming.source,
|
||||
authIndex: incoming.authIndex,
|
||||
inputTokens: incoming.inputTokens,
|
||||
outputTokens: incoming.outputTokens,
|
||||
cacheReadTokens: incoming.cacheReadTokens,
|
||||
failed: incoming.failed,
|
||||
});
|
||||
|
||||
expect(legacyDetail?.requestCount).toBe(1);
|
||||
expect(Number.isFinite(legacyDetail?.cost)).toBe(true);
|
||||
|
||||
const merged = mergeCliproxyUsageHistoryDetails([legacyDetail!], [incoming]);
|
||||
expect(merged).toHaveLength(1);
|
||||
expect(merged[0].provider).toBe(incoming.provider);
|
||||
expect(merged[0].cost).toBe(incoming.cost);
|
||||
});
|
||||
|
||||
it('preserves legitimate duplicate requests when the incoming batch has more occurrences', () => {
|
||||
const details = extractCliproxyUsageHistoryDetails(sampleResponse);
|
||||
const repeated = [details[0], { ...details[0] }];
|
||||
|
||||
Reference in New Issue
Block a user