feat(cliproxy): self-pause exhausted quota accounts

This commit is contained in:
Tam Nhu Tran
2026-04-28 20:35:15 -04:00
parent 849bf85755
commit 5013d411a7
15 changed files with 1000 additions and 113 deletions
+1
View File
@@ -41,6 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic
### Recent Fixes ### Recent Fixes
- **2026-04-28**: **#1123** CLIProxy quota failover now uses the dashboard/manual pause mechanism for all quota-visible OAuth providers with CCS quota fetchers: Antigravity, Claude, Codex, Gemini CLI, and GitHub Copilot. When a healthy fallback exists, CCS moves the exhausted account token out of the live `auth/` folder into `auth-paused/`, marks the account paused for dashboard visibility, persists the cooldown for auto-resume, and still avoids self-pausing the last usable account.
- **2026-04-28**: **#1115** CCS now exposes upstream CLIProxy session affinity as a first-class local managed setting. Users can inspect and toggle local `session-affinity` plus TTL from `ccs cliproxy routing affinity`, from the `/cliproxy` dashboard routing card, and through the local dashboard API. The generated local CLIProxy config now persists `routing.session-affinity` and `routing.session-affinity-ttl`, help/copy explains that CLIProxy prefers explicit session or thread identifiers before falling back to prompt-history hashing, and remote session-affinity management stays explicitly unsupported until upstream management APIs expose more than `routing.strategy`. - **2026-04-28**: **#1115** CCS now exposes upstream CLIProxy session affinity as a first-class local managed setting. Users can inspect and toggle local `session-affinity` plus TTL from `ccs cliproxy routing affinity`, from the `/cliproxy` dashboard routing card, and through the local dashboard API. The generated local CLIProxy config now persists `routing.session-affinity` and `routing.session-affinity-ttl`, help/copy explains that CLIProxy prefers explicit session or thread identifiers before falling back to prompt-history hashing, and remote session-affinity management stays explicitly unsupported until upstream management APIs expose more than `routing.strategy`.
- **2026-04-24**: **#1065** Local CLIProxy Plus is available again as an explicit opt-in backend through the community-maintained `kaitranntt/CLIProxyAPIPlus` fork. CCS keeps `original` as the default backend, no longer downgrades saved `backend: plus` configs to `original`, updates Plus release lookups to the maintained fork, and documents Plus as a targeted path for plus-only providers. - **2026-04-24**: **#1065** Local CLIProxy Plus is available again as an explicit opt-in backend through the community-maintained `kaitranntt/CLIProxyAPIPlus` fork. CCS keeps `original` as the default backend, no longer downgrades saved `backend: plus` configs to `original`, updates Plus release lookups to the maintained fork, and documents Plus as a targeted path for plus-only providers.
- **2026-04-21**: CLIProxy quota failover now quarantines exhausted Claude and Antigravity accounts out of live rotation when a healthy fallback exists. CCS persists those quota-triggered pauses across launches, automatically resumes them after the configured cooldown window, and deliberately avoids auto-pausing the last available account so single-account setups still degrade gracefully instead of hard-locking themselves. - **2026-04-21**: CLIProxy quota failover now quarantines exhausted Claude and Antigravity accounts out of live rotation when a healthy fallback exists. CCS persists those quota-triggered pauses across launches, automatically resumes them after the configured cooldown window, and deliberately avoids auto-pausing the last available account so single-account setups still degrade gracefully instead of hard-locking themselves.
+6 -4
View File
@@ -261,7 +261,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise<bool
### Overview ### Overview
Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account. Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account.
When CCS detects exhaustion and a healthy fallback exists, it also temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
``` ```
+===========================================================================+ +===========================================================================+
@@ -287,9 +287,11 @@ When CCS detects exhaustion and a healthy fallback exists, it also temporarily p
| +---> Check isPaused flag --> Skip if paused | +---> Check isPaused flag --> Skip if paused
| | | |
| +---> Fetch quota from provider API | +---> Fetch quota from provider API
| | - Gemini: /models endpoint | | - Antigravity: fetchAvailableModels
| | - Codex: /api/v1/account | | - Claude: policy limits endpoint
| | - Kiro: /api/usage | | - Codex: ChatGPT usage windows
| | - Gemini CLI: Code Assist quota buckets
| | - GitHub Copilot: copilot_internal/user snapshots
| | | |
| +---> Detect tier (free/paid/unknown) | +---> Detect tier (free/paid/unknown)
| | | |
+6 -1
View File
@@ -690,6 +690,9 @@ export async function handleQuotaExhaustion(
// Dynamic imports to avoid circular dependencies // Dynamic imports to avoid circular dependencies
const { applyCooldown, findHealthyAccount } = await import('./quota-manager'); const { applyCooldown, findHealthyAccount } = await import('./quota-manager');
const { setDefaultAccount, touchAccount } = await import('./account-manager'); const { setDefaultAccount, touchAccount } = await import('./account-manager');
const { loadOrCreateUnifiedConfig } = await import('../config/unified-config-loader');
const config = loadOrCreateUnifiedConfig();
const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5;
// Apply cooldown to exhausted account // Apply cooldown to exhausted account
applyCooldown(provider, accountId, cooldownMinutes); applyCooldown(provider, accountId, cooldownMinutes);
@@ -698,7 +701,9 @@ export async function handleQuotaExhaustion(
const alternative = await findHealthyAccount(provider, [accountId]); const alternative = await findHealthyAccount(provider, [accountId]);
if (alternative) { if (alternative) {
pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes); if (alternative.lastQuota !== null && alternative.lastQuota >= threshold) {
pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes);
}
setDefaultAccount(provider, alternative.id); setDefaultAccount(provider, alternative.id);
touchAccount(provider, alternative.id); touchAccount(provider, alternative.id);
writeQuotaExhausted(accountId, alternative.id, cooldownMinutes); writeQuotaExhausted(accountId, alternative.id, cooldownMinutes);
+33 -5
View File
@@ -106,6 +106,7 @@ import {
handleTokenExpiration, handleTokenExpiration,
handleQuotaCheck, handleQuotaCheck,
} from './retry-handler'; } from './retry-handler';
import { MANAGED_QUOTA_PROVIDERS, type ManagedQuotaProvider } from '../quota-manager';
import { checkOrJoinProxy, registerProxySession, setupCleanupHandlers } from './session-bridge'; import { checkOrJoinProxy, registerProxySession, setupCleanupHandlers } from './session-bridge';
import { parseThinkingOverride } from './thinking-arg-parser'; import { parseThinkingOverride } from './thinking-arg-parser';
import { import {
@@ -128,6 +129,25 @@ import {
} from './thinking-override-resolver'; } from './thinking-override-resolver';
import { shouldStartHttpsTunnel } from './https-tunnel-policy'; import { shouldStartHttpsTunnel } from './https-tunnel-policy';
function resolveRuntimeQuotaMonitorProviders(
provider: CLIProxyProvider,
compositeProviders: CLIProxyProvider[]
): ManagedQuotaProvider[] {
const candidates = compositeProviders.length > 0 ? compositeProviders : [provider];
const resolved: ManagedQuotaProvider[] = [];
for (const candidate of candidates) {
if (
MANAGED_QUOTA_PROVIDERS.includes(candidate as ManagedQuotaProvider) &&
!resolved.includes(candidate as ManagedQuotaProvider)
) {
resolved.push(candidate as ManagedQuotaProvider);
}
}
return resolved;
}
/** Default executor configuration */ /** Default executor configuration */
const DEFAULT_CONFIG: ExecutorConfig = { const DEFAULT_CONFIG: ExecutorConfig = {
port: CLIPROXY_DEFAULT_PORT, port: CLIPROXY_DEFAULT_PORT,
@@ -888,8 +908,7 @@ export async function execClaudeWithCLIProxy(
if (!skipLocalAuth) { if (!skipLocalAuth) {
// Multi-tier quota check for composite variants (check if any tier uses a managed provider) // Multi-tier quota check for composite variants (check if any tier uses a managed provider)
if (compositeProviders.length > 0) { if (compositeProviders.length > 0) {
const managedQuotaProviders = ['agy', 'claude'] as const; for (const managedProvider of MANAGED_QUOTA_PROVIDERS) {
for (const managedProvider of managedQuotaProviders) {
if (compositeProviders.includes(managedProvider)) { if (compositeProviders.includes(managedProvider)) {
await handleQuotaCheck(managedProvider); await handleQuotaCheck(managedProvider);
} }
@@ -1376,9 +1395,14 @@ export async function execClaudeWithCLIProxy(
// 12b. Start runtime quota monitor (adaptive polling during session) // 12b. Start runtime quota monitor (adaptive polling during session)
if (!skipLocalAuth) { if (!skipLocalAuth) {
const { startQuotaMonitor } = await import('../quota-manager'); const { startQuotaMonitor } = await import('../quota-manager');
const monitorAccount = getDefaultAccount(provider); for (const monitorProvider of resolveRuntimeQuotaMonitorProviders(
if (monitorAccount) { provider,
startQuotaMonitor(provider, monitorAccount.id); compositeProviders
)) {
const monitorAccount = getDefaultAccount(monitorProvider);
if (monitorAccount) {
startQuotaMonitor(monitorProvider, monitorAccount.id);
}
} }
} }
@@ -1397,4 +1421,8 @@ export async function execClaudeWithCLIProxy(
// Re-export utility functions for backwards compatibility // Re-export utility functions for backwards compatibility
export { isPortAvailable, findAvailablePort } from './lifecycle-manager'; export { isPortAvailable, findAvailablePort } from './lifecycle-manager';
export const __testExports = {
resolveRuntimeQuotaMonitorProviders,
};
export default execClaudeWithCLIProxy; export default execClaudeWithCLIProxy;
+2 -2
View File
@@ -80,9 +80,9 @@ export async function handleTokenExpiration(
* Handle quota check and auto-switching for providers with quota-based rotation. * Handle quota check and auto-switching for providers with quota-based rotation.
*/ */
export async function handleQuotaCheck(provider: CLIProxyProvider): Promise<void> { export async function handleQuotaCheck(provider: CLIProxyProvider): Promise<void> {
if (provider !== 'agy' && provider !== 'claude') return; const { isManagedQuotaProvider, preflightCheck } = await import('../quota-manager');
if (!isManagedQuotaProvider(provider)) return;
const { preflightCheck } = await import('../quota-manager');
const preflight = await preflightCheck(provider); const preflight = await preflightCheck(provider);
if (!preflight.proceed) { if (!preflight.proceed) {
+20 -8
View File
@@ -48,29 +48,41 @@ interface TokenData {
} }
function normalizeSnapshot(raw?: RawGhcpQuotaSnapshot): GhcpQuotaSnapshot { function normalizeSnapshot(raw?: RawGhcpQuotaSnapshot): GhcpQuotaSnapshot {
const unlimited = Boolean(raw?.unlimited);
const hasPercentSignal =
typeof raw?.percent_remaining === 'number' && Number.isFinite(raw.percent_remaining);
const rawRemaining = typeof raw?.remaining === 'number' ? raw.remaining : raw?.quota_remaining;
const hasEntitlementRemainingSignal =
typeof raw?.entitlement === 'number' &&
Number.isFinite(raw.entitlement) &&
raw.entitlement > 0 &&
typeof rawRemaining === 'number' &&
Number.isFinite(rawRemaining);
const reported =
raw !== undefined && (unlimited || hasPercentSignal || hasEntitlementRemainingSignal);
const entitlement = Number(raw?.entitlement ?? 0); const entitlement = Number(raw?.entitlement ?? 0);
const remainingRaw = raw?.remaining ?? raw?.quota_remaining ?? 0; const remainingRaw = rawRemaining ?? 0;
const remaining = Number(remainingRaw); const remaining = Number(remainingRaw);
const safeEntitlement = Number.isFinite(entitlement) ? Math.max(0, entitlement) : 0; const safeEntitlement = Number.isFinite(entitlement) ? Math.max(0, entitlement) : 0;
const safeRemaining = Number.isFinite(remaining) ? Math.max(0, remaining) : 0; const safeRemaining = Number.isFinite(remaining) ? Math.max(0, remaining) : 0;
const used = Math.max(0, safeEntitlement - safeRemaining); const used = Math.max(0, safeEntitlement - safeRemaining);
const percentRemainingRaw = const percentRemaining = unlimited
typeof raw?.percent_remaining === 'number' ? raw.percent_remaining : null; ? 100
const percentRemaining = : hasPercentSignal
percentRemainingRaw !== null ? clampPercent(raw.percent_remaining as number)
? clampPercent(percentRemainingRaw) : reported && safeEntitlement > 0
: safeEntitlement > 0
? clampPercent((safeRemaining / safeEntitlement) * 100) ? clampPercent((safeRemaining / safeEntitlement) * 100)
: 0; : 0;
return { return {
reported,
entitlement: safeEntitlement, entitlement: safeEntitlement,
remaining: safeRemaining, remaining: safeRemaining,
used, used,
percentRemaining, percentRemaining,
percentUsed: clampPercent(100 - percentRemaining), percentUsed: clampPercent(100 - percentRemaining),
unlimited: Boolean(raw?.unlimited), unlimited,
overageCount: overageCount:
typeof raw?.overage_count === 'number' && Number.isFinite(raw.overage_count) typeof raw?.overage_count === 'number' && Number.isFinite(raw.overage_count)
? Math.max(0, raw.overage_count) ? Math.max(0, raw.overage_count)
+219 -67
View File
@@ -15,7 +15,15 @@
import { CLIProxyProvider } from './types'; import { CLIProxyProvider } from './types';
import { QuotaResult, fetchAccountQuota } from './quota-fetcher'; import { QuotaResult, fetchAccountQuota } from './quota-fetcher';
import { fetchClaudeQuota } from './quota-fetcher-claude'; import { fetchClaudeQuota } from './quota-fetcher-claude';
import type { ClaudeQuotaResult } from './quota-types'; import { fetchCodexQuota } from './quota-fetcher-codex';
import { fetchGeminiCliQuota } from './quota-fetcher-gemini-cli';
import { fetchGhcpQuota } from './quota-fetcher-ghcp';
import type {
ClaudeQuotaResult,
CodexQuotaResult,
GeminiCliQuotaResult,
GhcpQuotaResult,
} from './quota-types';
import { import {
getDefaultAccount, getDefaultAccount,
getProviderAccounts, getProviderAccounts,
@@ -27,12 +35,25 @@ import {
import { loadOrCreateUnifiedConfig } from '../config/unified-config-loader'; import { loadOrCreateUnifiedConfig } from '../config/unified-config-loader';
import type { RuntimeMonitorConfig } from '../config/unified-config-types'; import type { RuntimeMonitorConfig } from '../config/unified-config-types';
type ManagedQuotaProvider = 'agy' | 'claude'; export type ManagedQuotaProvider = 'agy' | 'claude' | 'codex' | 'gemini' | 'ghcp';
type ManagedQuotaResult = QuotaResult | ClaudeQuotaResult; type ManagedQuotaResult =
| QuotaResult
| ClaudeQuotaResult
| CodexQuotaResult
| GeminiCliQuotaResult
| GhcpQuotaResult;
const MANAGED_QUOTA_PROVIDERS: readonly ManagedQuotaProvider[] = ['agy', 'claude']; export const MANAGED_QUOTA_PROVIDERS: readonly ManagedQuotaProvider[] = [
'agy',
'claude',
'codex',
'gemini',
'ghcp',
];
function isManagedQuotaProvider(provider: CLIProxyProvider): provider is ManagedQuotaProvider { export function isManagedQuotaProvider(
provider: CLIProxyProvider
): provider is ManagedQuotaProvider {
return MANAGED_QUOTA_PROVIDERS.includes(provider as ManagedQuotaProvider); return MANAGED_QUOTA_PROVIDERS.includes(provider as ManagedQuotaProvider);
} }
@@ -117,19 +138,7 @@ async function fetchQuotaWithDedup(
setCachedQuota(provider, accountId, result); setCachedQuota(provider, accountId, result);
return result; return result;
}) })
.catch((): ManagedQuotaResult => { .catch((): ManagedQuotaResult => buildFailedQuotaResult(provider, accountId))
if (provider === 'claude') {
return {
success: false,
windows: [],
coreUsage: { fiveHour: null, weekly: null },
lastUpdated: Date.now(),
error: 'Failed to fetch Claude quota',
accountId,
};
}
return { success: false, models: [], lastUpdated: Date.now() };
})
.finally(() => { .finally(() => {
pendingFetches.delete(key); pendingFetches.delete(key);
}); });
@@ -143,10 +152,85 @@ async function fetchManagedQuota(
accountId: string, accountId: string,
verbose: boolean verbose: boolean
): Promise<ManagedQuotaResult> { ): Promise<ManagedQuotaResult> {
if (provider === 'claude') { switch (provider) {
return fetchClaudeQuota(accountId, verbose); case 'agy':
return fetchAccountQuota(provider, accountId, verbose);
case 'claude':
return fetchClaudeQuota(accountId, verbose);
case 'codex':
return fetchCodexQuota(accountId, verbose);
case 'gemini':
return fetchGeminiCliQuota(accountId, verbose);
case 'ghcp':
return fetchGhcpQuota(accountId, verbose);
}
}
function buildFailedGhcpSnapshot(): GhcpQuotaResult['snapshots']['chat'] {
return {
reported: false,
entitlement: 0,
remaining: 0,
used: 0,
percentRemaining: 0,
percentUsed: 0,
unlimited: false,
overageCount: 0,
overagePermitted: false,
quotaId: null,
};
}
function buildFailedQuotaResult(
provider: ManagedQuotaProvider,
accountId: string
): ManagedQuotaResult {
switch (provider) {
case 'agy':
return { success: false, models: [], lastUpdated: Date.now(), accountId };
case 'claude':
return {
success: false,
windows: [],
coreUsage: { fiveHour: null, weekly: null },
lastUpdated: Date.now(),
error: 'Failed to fetch Claude quota',
accountId,
};
case 'codex':
return {
success: false,
windows: [],
coreUsage: { fiveHour: null, weekly: null },
planType: null,
lastUpdated: Date.now(),
error: 'Failed to fetch Codex quota',
accountId,
};
case 'gemini':
return {
success: false,
buckets: [],
projectId: null,
lastUpdated: Date.now(),
error: 'Failed to fetch Gemini quota',
accountId,
};
case 'ghcp':
return {
success: false,
planType: null,
quotaResetDate: null,
snapshots: {
premiumInteractions: buildFailedGhcpSnapshot(),
chat: buildFailedGhcpSnapshot(),
completions: buildFailedGhcpSnapshot(),
},
lastUpdated: Date.now(),
error: 'Failed to fetch GitHub Copilot quota',
accountId,
};
} }
return fetchAccountQuota(provider, accountId, verbose);
} }
// ============================================================================ // ============================================================================
@@ -264,14 +348,62 @@ function calculateClaudeQuotaPercent(quota: ClaudeQuotaResult): number | null {
return null; return null;
} }
function calculateCodexQuotaPercent(quota: CodexQuotaResult): number | null {
if (!quota.success) return null;
const coreWindows = [quota.coreUsage?.fiveHour, quota.coreUsage?.weekly].filter(
(window): window is NonNullable<typeof window> => !!window
);
if (coreWindows.length > 0) {
return Math.min(...coreWindows.map((window) => window.remainingPercent));
}
const usageWindows = quota.windows.filter((window) => window.category !== 'code-review');
if (usageWindows.length > 0) {
return Math.min(...usageWindows.map((window) => window.remainingPercent));
}
return null;
}
function calculateGeminiQuotaPercent(quota: GeminiCliQuotaResult): number | null {
if (!quota.success || quota.buckets.length === 0) return null;
return Math.min(...quota.buckets.map((bucket) => bucket.remainingPercent));
}
function calculateGhcpQuotaPercent(quota: GhcpQuotaResult): number | null {
if (!quota.success) return null;
const percentages = [
quota.snapshots.premiumInteractions,
quota.snapshots.chat,
quota.snapshots.completions,
]
.filter((snapshot) => snapshot.reported !== false)
.map((snapshot) => (snapshot.unlimited ? 100 : snapshot.percentRemaining))
.filter((percentage) => Number.isFinite(percentage));
return percentages.length > 0 ? Math.min(...percentages) : null;
}
/** /**
* Calculate normalized quota percentage for managed providers. * Calculate normalized quota percentage for managed providers.
*/ */
function calculateQuotaPercent(quota: ManagedQuotaResult): number | null { function calculateQuotaPercent(
if ('models' in quota) { provider: ManagedQuotaProvider,
return calculateAgyQuotaPercent(quota); quota: ManagedQuotaResult
): number | null {
switch (provider) {
case 'agy':
return calculateAgyQuotaPercent(quota as QuotaResult);
case 'claude':
return calculateClaudeQuotaPercent(quota as ClaudeQuotaResult);
case 'codex':
return calculateCodexQuotaPercent(quota as CodexQuotaResult);
case 'gemini':
return calculateGeminiQuotaPercent(quota as GeminiCliQuotaResult);
case 'ghcp':
return calculateGhcpQuotaPercent(quota as GhcpQuotaResult);
} }
return calculateClaudeQuotaPercent(quota);
} }
/** /**
@@ -312,7 +444,7 @@ export async function findHealthyAccount(
quota = await fetchQuotaWithDedup(provider, account.id); quota = await fetchQuotaWithDedup(provider, account.id);
} }
const avgQuota = calculateQuotaPercent(quota); const avgQuota = calculateQuotaPercent(provider, quota);
return { return {
id: account.id, id: account.id,
@@ -437,7 +569,7 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
// Calculate normalized quota percentage. // Calculate normalized quota percentage.
// Null means quota data is unavailable (e.g. provider does not expose limits for this account). // Null means quota data is unavailable (e.g. provider does not expose limits for this account).
const quotaPercent = calculateQuotaPercent(quota); const quotaPercent = calculateQuotaPercent(provider, quota);
if (quotaPercent === null) { if (quotaPercent === null) {
return { return {
proceed: true, proceed: true,
@@ -462,11 +594,13 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
}; };
} }
pauseAccountForQuotaCooldown( if (alternative.lastQuota !== null && alternative.lastQuota >= threshold) {
provider, pauseAccountForQuotaCooldown(
defaultAccount.id, provider,
quotaConfig.auto?.cooldown_minutes ?? 5 defaultAccount.id,
); quotaConfig.auto?.cooldown_minutes ?? 5
);
}
setDefaultAccount(provider, alternative.id); setDefaultAccount(provider, alternative.id);
touchAccount(provider, alternative.id); touchAccount(provider, alternative.id);
return { return {
@@ -506,12 +640,14 @@ export async function getQuotaStatus(provider: CLIProxyProvider): Promise<{
const results = await Promise.all( const results = await Promise.all(
accounts.map(async (account) => { accounts.map(async (account) => {
const managedProvider = isManagedQuotaProvider(provider) ? provider : null;
let quota = getCachedQuota(provider, account.id); let quota = getCachedQuota(provider, account.id);
if (!quota && isManagedQuotaProvider(provider)) { if (!quota && managedProvider) {
quota = await fetchQuotaWithDedup(provider, account.id); quota = await fetchQuotaWithDedup(managedProvider, account.id);
} }
const avgQuota = quota ? calculateQuotaPercent(quota) : null; const avgQuota =
quota && managedProvider ? calculateQuotaPercent(managedProvider, quota) : null;
return { return {
account, account,
@@ -530,14 +666,14 @@ export async function getQuotaStatus(provider: CLIProxyProvider): Promise<{
// RUNTIME QUOTA MONITOR (adaptive polling during active sessions) // RUNTIME QUOTA MONITOR (adaptive polling during active sessions)
// ============================================================================ // ============================================================================
/** Active monitor timer (null = not running) */ interface RuntimeMonitorState {
let monitorTimer: ReturnType<typeof setTimeout> | null = null; timer: ReturnType<typeof setTimeout> | null;
hasWarnedThisSession: boolean;
stopped: boolean;
}
/** Tracks if warning was shown this session (avoid spam) */ /** Active monitor timers keyed by provider/account. */
let hasWarnedThisSession = false; const activeRuntimeMonitors = new Map<string, RuntimeMonitorState>();
/** Guards against in-flight poll callbacks running after stop */
let monitorStopped = false;
/** /**
* Schedule next quota poll with adaptive interval. * Schedule next quota poll with adaptive interval.
@@ -547,16 +683,18 @@ function scheduleNextPoll(
provider: ManagedQuotaProvider, provider: ManagedQuotaProvider,
accountId: string, accountId: string,
monitorConfig: RuntimeMonitorConfig, monitorConfig: RuntimeMonitorConfig,
intervalMs: number intervalMs: number,
state: RuntimeMonitorState
): void { ): void {
monitorTimer = setTimeout(async () => { const monitorKey = getCacheKey(provider, accountId);
state.timer = setTimeout(async () => {
// Guard: skip if monitor was stopped while this callback was queued // Guard: skip if monitor was stopped while this callback was queued
if (monitorStopped) return; if (state.stopped || activeRuntimeMonitors.get(monitorKey) !== state) return;
try { try {
const quota = await fetchQuotaWithDedup(provider, accountId); const quota = await fetchQuotaWithDedup(provider, accountId);
if (monitorStopped) return; // Re-check after async fetch if (state.stopped || activeRuntimeMonitors.get(monitorKey) !== state) return;
const avgQuota = calculateQuotaPercent(quota); const avgQuota = calculateQuotaPercent(provider, quota);
if (avgQuota === null) { if (avgQuota === null) {
// Quota data unavailable: keep polling, but do not treat unknown as healthy/exhausted. // Quota data unavailable: keep polling, but do not treat unknown as healthy/exhausted.
@@ -564,7 +702,8 @@ function scheduleNextPoll(
provider, provider,
accountId, accountId,
monitorConfig, monitorConfig,
monitorConfig.normal_interval_seconds * 1000 monitorConfig.normal_interval_seconds * 1000,
state
); );
return; return;
} }
@@ -576,22 +715,25 @@ function scheduleNextPoll(
// default only takes effect on next session start via preflightCheck(). // default only takes effect on next session start via preflightCheck().
const { handleQuotaExhaustion } = await import('./account-safety'); const { handleQuotaExhaustion } = await import('./account-safety');
await handleQuotaExhaustion(provider, accountId, monitorConfig.cooldown_minutes); await handleQuotaExhaustion(provider, accountId, monitorConfig.cooldown_minutes);
monitorTimer = null; state.stopped = true;
state.timer = null;
activeRuntimeMonitors.delete(monitorKey);
return; // Stop polling return; // Stop polling
} }
if (avgQuota <= monitorConfig.warn_threshold) { if (avgQuota <= monitorConfig.warn_threshold) {
// WARNING: switch to critical interval, warn once // WARNING: switch to critical interval, warn once
if (!hasWarnedThisSession) { if (!state.hasWarnedThisSession) {
const { writeQuotaWarning } = await import('./account-safety'); const { writeQuotaWarning } = await import('./account-safety');
writeQuotaWarning(accountId, avgQuota); writeQuotaWarning(accountId, avgQuota);
hasWarnedThisSession = true; state.hasWarnedThisSession = true;
} }
scheduleNextPoll( scheduleNextPoll(
provider, provider,
accountId, accountId,
monitorConfig, monitorConfig,
monitorConfig.critical_interval_seconds * 1000 monitorConfig.critical_interval_seconds * 1000,
state
); );
return; return;
} }
@@ -601,17 +743,20 @@ function scheduleNextPoll(
provider, provider,
accountId, accountId,
monitorConfig, monitorConfig,
monitorConfig.normal_interval_seconds * 1000 monitorConfig.normal_interval_seconds * 1000,
state
); );
} catch { } catch {
// API failure: silently reschedule at same interval // API failure: silently reschedule at same interval
scheduleNextPoll(provider, accountId, monitorConfig, intervalMs); if (!state.stopped && activeRuntimeMonitors.get(monitorKey) === state) {
scheduleNextPoll(provider, accountId, monitorConfig, intervalMs, state);
}
} }
}, intervalMs); }, intervalMs);
// Prevent monitor from keeping Node.js process alive // Prevent monitor from keeping Node.js process alive
if (monitorTimer && typeof monitorTimer === 'object' && 'unref' in monitorTimer) { if (state.timer && typeof state.timer === 'object' && 'unref' in state.timer) {
monitorTimer.unref(); state.timer.unref();
} }
} }
@@ -628,9 +773,6 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
// Only managed providers support runtime quota monitoring. // Only managed providers support runtime quota monitoring.
if (!isManagedQuotaProvider(provider)) return; if (!isManagedQuotaProvider(provider)) return;
// Prevent duplicate monitors
if (monitorTimer) return;
const config = loadOrCreateUnifiedConfig(); const config = loadOrCreateUnifiedConfig();
const quotaConfig = config.quota_management; const quotaConfig = config.quota_management;
@@ -647,15 +789,23 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
return; // Invalid config — skip monitoring silently (logged at config level) return; // Invalid config — skip monitoring silently (logged at config level)
} }
hasWarnedThisSession = false; const monitorKey = getCacheKey(provider, accountId);
monitorStopped = false; if (activeRuntimeMonitors.has(monitorKey)) return;
const state: RuntimeMonitorState = {
timer: null,
hasWarnedThisSession: false,
stopped: false,
};
activeRuntimeMonitors.set(monitorKey, state);
// Start first poll at normal interval // Start first poll at normal interval
scheduleNextPoll( scheduleNextPoll(
provider, provider,
accountId, accountId,
quotaConfig.runtime_monitor, quotaConfig.runtime_monitor,
quotaConfig.runtime_monitor.normal_interval_seconds * 1000 quotaConfig.runtime_monitor.normal_interval_seconds * 1000,
state
); );
} }
@@ -663,10 +813,12 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
* Stop the runtime quota monitor. Safe to call multiple times. * Stop the runtime quota monitor. Safe to call multiple times.
*/ */
export function stopQuotaMonitor(): void { export function stopQuotaMonitor(): void {
monitorStopped = true; for (const state of activeRuntimeMonitors.values()) {
if (monitorTimer) { state.stopped = true;
clearTimeout(monitorTimer); if (state.timer) {
monitorTimer = null; clearTimeout(state.timer);
state.timer = null;
}
} }
hasWarnedThisSession = false; activeRuntimeMonitors.clear();
} }
+2
View File
@@ -230,6 +230,8 @@ export interface GeminiCliQuotaResult extends QuotaErrorMetadata {
* GitHub Copilot quota snapshot. * GitHub Copilot quota snapshot.
*/ */
export interface GhcpQuotaSnapshot { export interface GhcpQuotaSnapshot {
/** False when upstream omitted this quota category; callers should exclude it from health math */
reported?: boolean;
/** Total quota allocation for this category */ /** Total quota allocation for this category */
entitlement: number; entitlement: number;
/** Remaining quota count */ /** Remaining quota count */
+19 -13
View File
@@ -752,13 +752,17 @@ function displayGhcpQuotaSection(results: { account: string; quota: GhcpQuotaRes
continue; continue;
} }
const rows = [ const reportedSnapshots = [
quota.snapshots.premiumInteractions.percentRemaining, quota.snapshots.premiumInteractions,
quota.snapshots.chat.percentRemaining, quota.snapshots.chat,
quota.snapshots.completions.percentRemaining, quota.snapshots.completions,
]; ].filter((snapshot) => snapshot.reported !== false);
const minQuota = rows.length > 0 ? Math.min(...rows) : 0; const rows = reportedSnapshots.map((snapshot) =>
const statusIcon = minQuota > 50 ? ok('') : minQuota > 10 ? warn('') : fail(''); snapshot.unlimited ? 100 : snapshot.percentRemaining
);
const minQuota = rows.length > 0 ? Math.min(...rows) : null;
const statusIcon =
minQuota === null ? info('') : minQuota > 50 ? ok('') : minQuota > 10 ? warn('') : fail('');
const planBadge = quota.planType ? color(` [${quota.planType}]`, 'info') : ''; const planBadge = quota.planType ? color(` [${quota.planType}]`, 'info') : '';
console.log(` ${statusIcon}${accountLabel}${defaultMark}${planBadge}`); console.log(` ${statusIcon}${accountLabel}${defaultMark}${planBadge}`);
@@ -766,12 +770,14 @@ function displayGhcpQuotaSection(results: { account: string; quota: GhcpQuotaRes
console.log(` ${dim(`Resets ${formatResetTimeISO(quota.quotaResetDate)}`)}`); console.log(` ${dim(`Resets ${formatResetTimeISO(quota.quotaResetDate)}`)}`);
} }
const items: Array<[string, GhcpQuotaResult['snapshots'][keyof GhcpQuotaResult['snapshots']]]> = const allItems: Array<
[ [string, GhcpQuotaResult['snapshots'][keyof GhcpQuotaResult['snapshots']]]
['Premium interactions', quota.snapshots.premiumInteractions], > = [
['Chat', quota.snapshots.chat], ['Premium interactions', quota.snapshots.premiumInteractions],
['Completions', quota.snapshots.completions], ['Chat', quota.snapshots.chat],
]; ['Completions', quota.snapshots.completions],
];
const items = allItems.filter(([, snapshot]) => snapshot.reported !== false);
for (const [label, snapshot] of items) { for (const [label, snapshot] of items) {
const bar = formatQuotaBar(snapshot.percentRemaining); const bar = formatQuotaBar(snapshot.percentRemaining);
@@ -86,6 +86,36 @@ function writeClaudeAuth(accountId: string, accessToken: string): void {
); );
} }
function writeCodexAuth(tokenFile: string, accountId: string, accessToken: string): void {
writeAuthToken(tokenFile, {
access_token: accessToken,
account_id: `chatgpt-${accountId}`,
expired: '2099-01-01T00:00:00.000Z',
type: 'codex',
email: accountId,
});
}
function writeGhcpAuth(tokenFile: string, accessToken: string): void {
writeAuthToken(tokenFile, {
access_token: accessToken,
type: 'ghcp',
});
}
function writeGeminiAuth(tokenFile: string, accountId: string, accessToken: string): void {
writeAuthToken(tokenFile, {
type: 'gemini',
email: accountId,
project_id: 'cloudaicompanion-test-123',
token: {
access_token: accessToken,
refresh_token: `${accessToken}-refresh`,
expiry: Date.now() + 60 * 60 * 1000,
},
});
}
function writeAuthToken(tokenFile: string, payload: Record<string, unknown>): void { function writeAuthToken(tokenFile: string, payload: Record<string, unknown>): void {
const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth'); const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth');
fs.mkdirSync(authDir, { recursive: true }); fs.mkdirSync(authDir, { recursive: true });
@@ -264,7 +294,7 @@ describe('Quota Exhaustion Handlers', () => {
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false); expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
}); });
it('should switch Claude accounts when fallback quota endpoint returns 404', async () => { it('should switch Claude accounts without durable pause when fallback quota is unknown', async () => {
writeRegistry({ writeRegistry({
claude: { claude: {
default: 'exhausted@example.com', default: 'exhausted@example.com',
@@ -312,11 +342,27 @@ describe('Quota Exhaustion Handlers', () => {
}) as typeof fetch; }) as typeof fetch;
const result = await handleQuotaExhaustion('claude', 'exhausted@example.com', 10); const result = await handleQuotaExhaustion('claude', 'exhausted@example.com', 10);
const { getDefaultAccount } = await import('../../../src/cliproxy/account-manager'); const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedTo).toBe('fallback@example.com'); expect(result.switchedTo).toBe('fallback@example.com');
expect(getDefaultAccount('claude')?.id).toBe('fallback@example.com'); expect(getDefaultAccount('claude')?.id).toBe('fallback@example.com');
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true); expect(getAccount('claude', 'exhausted@example.com')?.paused).not.toBe(true);
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(
false
);
expect(
fs.existsSync(
path.join(
tmpDir,
'.ccs',
'cliproxy',
'auth',
`claude-${sanitizeEmail('exhausted@example.com')}.json`
)
)
).toBe(true);
expect( expect(
fs.existsSync( fs.existsSync(
path.join( path.join(
@@ -327,6 +373,512 @@ describe('Quota Exhaustion Handlers', () => {
`claude-${sanitizeEmail('exhausted@example.com')}.json` `claude-${sanitizeEmail('exhausted@example.com')}.json`
) )
) )
).toBe(false);
});
it('should self-pause exhausted Codex accounts when a healthy fallback exists', async () => {
writeRegistry({
codex: {
default: 'exhausted@example.com',
accounts: {
'exhausted@example.com': {
email: 'exhausted@example.com',
tokenFile: 'codex-exhausted.json',
},
'fallback@example.com': {
email: 'fallback@example.com',
tokenFile: 'codex-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth('codex-exhausted.json', 'exhausted@example.com', 'exhausted-token');
writeCodexAuth('codex-fallback.json', 'fallback@example.com', 'fallback-token');
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
const usedPercent = accountHeader === 'chatgpt-fallback@example.com' ? 10 : 100;
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const result = await handleQuotaExhaustion('codex', 'exhausted@example.com', 10);
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedTo).toBe('fallback@example.com');
expect(getDefaultAccount('codex')?.id).toBe('fallback@example.com');
expect(getAccount('codex', 'exhausted@example.com')?.paused).toBe(true);
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-exhausted.json'))
).toBe(true);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-exhausted.json'))
).toBe(false);
});
it('should not durably self-pause the only exhausted Codex account during preflight', async () => {
writeRegistry({
codex: {
default: 'only-codex@example.com',
accounts: {
'only-codex@example.com': {
email: 'only-codex@example.com',
tokenFile: 'codex-only.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth('codex-only.json', 'only-codex@example.com', 'only-token');
global.fetch = mock(() =>
Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: 100, reset_after_seconds: 3600 },
secondary_window: { used_percent: 100, reset_after_seconds: 604800 },
},
})
)
) as typeof fetch;
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
const result = await preflightCheck('codex');
const { getAccount } = await import('../../../src/cliproxy/account-manager');
expect(result.accountId).toBe('only-codex@example.com');
expect(result.switchedFrom).toBeUndefined();
expect(result.reason).toContain('no alternatives available');
expect(getAccount('codex', 'only-codex@example.com')?.paused).not.toBe(true);
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-only.json'))).toBe(
true
);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-only.json'))
).toBe(false);
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
});
it('should self-pause an exhausted Codex default during preflight when fallback exists', async () => {
writeRegistry({
codex: {
default: 'preflight-exhausted@example.com',
accounts: {
'preflight-exhausted@example.com': {
email: 'preflight-exhausted@example.com',
tokenFile: 'codex-preflight-exhausted.json',
},
'preflight-fallback@example.com': {
email: 'preflight-fallback@example.com',
tokenFile: 'codex-preflight-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth(
'codex-preflight-exhausted.json',
'preflight-exhausted@example.com',
'preflight-exhausted-token'
);
writeCodexAuth(
'codex-preflight-fallback.json',
'preflight-fallback@example.com',
'preflight-fallback-token'
);
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
const usedPercent = accountHeader === 'chatgpt-preflight-fallback@example.com' ? 10 : 100;
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
const result = await preflightCheck('codex');
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedFrom).toBe('preflight-exhausted@example.com');
expect(result.accountId).toBe('preflight-fallback@example.com');
expect(getDefaultAccount('codex')?.id).toBe('preflight-fallback@example.com');
expect(getAccount('codex', 'preflight-exhausted@example.com')?.paused).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-preflight-exhausted.json')
)
).toBe(true);
});
it('should not durably self-pause during preflight when fallback quota is unknown', async () => {
writeRegistry({
codex: {
default: 'preflight-unknown-exhausted@example.com',
accounts: {
'preflight-unknown-exhausted@example.com': {
email: 'preflight-unknown-exhausted@example.com',
tokenFile: 'codex-preflight-unknown-exhausted.json',
},
'preflight-unknown-fallback@example.com': {
email: 'preflight-unknown-fallback@example.com',
tokenFile: 'codex-preflight-unknown-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth(
'codex-preflight-unknown-exhausted.json',
'preflight-unknown-exhausted@example.com',
'preflight-unknown-exhausted-token'
);
writeCodexAuth(
'codex-preflight-unknown-fallback.json',
'preflight-unknown-fallback@example.com',
'preflight-unknown-fallback-token'
);
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
if (accountHeader === 'chatgpt-preflight-unknown-fallback@example.com') {
return Promise.resolve(new Response('', { status: 500 }));
}
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: 100, reset_after_seconds: 3600 },
secondary_window: { used_percent: 100, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
const result = await preflightCheck('codex');
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedFrom).toBe('preflight-unknown-exhausted@example.com');
expect(result.accountId).toBe('preflight-unknown-fallback@example.com');
expect(getDefaultAccount('codex')?.id).toBe('preflight-unknown-fallback@example.com');
expect(getAccount('codex', 'preflight-unknown-exhausted@example.com')?.paused).not.toBe(
true
);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-preflight-unknown-exhausted.json')
)
).toBe(true);
expect(
fs.existsSync(
path.join(
tmpDir,
'.ccs',
'cliproxy',
'auth-paused',
'codex-preflight-unknown-exhausted.json'
)
)
).toBe(false);
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(
false
);
});
it('should self-pause exhausted Gemini accounts when a healthy fallback exists', async () => {
writeRegistry({
gemini: {
default: 'gemini-exhausted@example.com',
accounts: {
'gemini-exhausted@example.com': {
email: 'gemini-exhausted@example.com',
tokenFile: 'gemini-exhausted.json',
},
'gemini-fallback@example.com': {
email: 'gemini-fallback@example.com',
tokenFile: 'gemini-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeGeminiAuth('gemini-exhausted.json', 'gemini-exhausted@example.com', 'exhausted-token');
writeGeminiAuth('gemini-fallback.json', 'gemini-fallback@example.com', 'fallback-token');
global.fetch = mock((_url: string, options?: RequestInit) => {
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
const remainingFraction = authHeader === 'Bearer fallback-token' ? 0.9 : 0;
return Promise.resolve(
Response.json({
buckets: [
{
model_id: 'gemini-3-flash-preview',
remaining_fraction: remainingFraction,
remaining_amount: Math.round(remainingFraction * 100),
reset_time: '2026-05-01T00:00:00Z',
},
],
})
);
}) as typeof fetch;
const result = await handleQuotaExhaustion('gemini', 'gemini-exhausted@example.com', 10);
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedTo).toBe('gemini-fallback@example.com');
expect(getDefaultAccount('gemini')?.id).toBe('gemini-fallback@example.com');
expect(getAccount('gemini', 'gemini-exhausted@example.com')?.paused).toBe(true);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'gemini-exhausted.json'))
).toBe(true);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'gemini-exhausted.json'))
).toBe(false);
});
it('should self-pause exhausted GitHub Copilot accounts when a healthy fallback exists', async () => {
writeRegistry({
ghcp: {
default: 'ghcp-exhausted',
accounts: {
'ghcp-exhausted': {
tokenFile: 'ghcp-exhausted.json',
},
'ghcp-fallback': {
tokenFile: 'ghcp-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeGhcpAuth('ghcp-exhausted.json', 'exhausted-token');
writeGhcpAuth('ghcp-fallback.json', 'fallback-token');
global.fetch = mock((_url: string, options?: RequestInit) => {
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
const remaining = authHeader === 'token fallback-token' ? 90 : 0;
return Promise.resolve(
Response.json({
copilot_plan: 'individual',
quota_reset_date: '2026-05-01T00:00:00Z',
quota_snapshots: {
premium_interactions: { entitlement: 100, remaining },
chat: { entitlement: 100, remaining },
completions: { entitlement: 100, remaining },
},
})
);
}) as typeof fetch;
const result = await handleQuotaExhaustion('ghcp', 'ghcp-exhausted', 10);
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedTo).toBe('ghcp-fallback');
expect(getDefaultAccount('ghcp')?.id).toBe('ghcp-fallback');
expect(getAccount('ghcp', 'ghcp-exhausted')?.paused).toBe(true);
expect(
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'ghcp-exhausted.json'))
).toBe(true);
});
it('should ignore omitted GitHub Copilot snapshots when selecting a fallback', async () => {
writeRegistry({
ghcp: {
default: 'ghcp-partial-exhausted',
accounts: {
'ghcp-partial-exhausted': {
tokenFile: 'ghcp-partial-exhausted.json',
},
'ghcp-partial-fallback': {
tokenFile: 'ghcp-partial-fallback.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeGhcpAuth('ghcp-partial-exhausted.json', 'partial-exhausted-token');
writeGhcpAuth('ghcp-partial-fallback.json', 'partial-fallback-token');
global.fetch = mock((_url: string, options?: RequestInit) => {
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
const remaining = authHeader === 'token partial-fallback-token' ? 90 : 0;
return Promise.resolve(
Response.json({
copilot_plan: 'individual',
quota_reset_date: '2026-05-01T00:00:00Z',
quota_snapshots: {
premium_interactions: { entitlement: 100, remaining },
chat: { entitlement: 100, remaining },
},
})
);
}) as typeof fetch;
const result = await handleQuotaExhaustion('ghcp', 'ghcp-partial-exhausted', 10);
const { getAccount, getDefaultAccount } = await import(
'../../../src/cliproxy/account-manager'
);
expect(result.switchedTo).toBe('ghcp-partial-fallback');
expect(getDefaultAccount('ghcp')?.id).toBe('ghcp-partial-fallback');
expect(getAccount('ghcp', 'ghcp-partial-exhausted')?.paused).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'ghcp-partial-exhausted.json')
)
).toBe(true); ).toBe(true);
}); });
@@ -76,6 +76,7 @@ describe('GHCP Quota Fetcher', () => {
const snapshot = normalizeGhcpSnapshot(); const snapshot = normalizeGhcpSnapshot();
expect(snapshot).toEqual({ expect(snapshot).toEqual({
reported: false,
entitlement: 0, entitlement: 0,
remaining: 0, remaining: 0,
used: 0, used: 0,
@@ -88,6 +89,50 @@ describe('GHCP Quota Fetcher', () => {
}); });
}); });
it('marks upstream-provided snapshots as reported', () => {
const snapshot = normalizeGhcpSnapshot({
entitlement: 100,
remaining: 80,
});
expect(snapshot.reported).toBe(true);
});
it('treats unknown upstream snapshot objects as unreported', () => {
const empty = normalizeGhcpSnapshot({});
const quotaIdOnly = normalizeGhcpSnapshot({ quota_id: 'chat' });
const remainingOnly = normalizeGhcpSnapshot({ remaining: 80 });
for (const snapshot of [empty, quotaIdOnly, remainingOnly]) {
expect(snapshot.reported).toBe(false);
expect(snapshot.percentRemaining).toBe(0);
expect(snapshot.percentUsed).toBe(100);
}
});
it('marks percent-only snapshots as reported', () => {
const snapshot = normalizeGhcpSnapshot({
percent_remaining: 70,
});
expect(snapshot.reported).toBe(true);
expect(snapshot.percentRemaining).toBe(70);
expect(snapshot.percentUsed).toBe(30);
});
it('treats unlimited snapshots as 100% remaining', () => {
const snapshot = normalizeGhcpSnapshot({
unlimited: true,
entitlement: 0,
remaining: 0,
});
expect(snapshot.reported).toBe(true);
expect(snapshot.percentRemaining).toBe(100);
expect(snapshot.percentUsed).toBe(0);
expect(snapshot.unlimited).toBe(true);
});
it('clamps percent_remaining to 0-100 range', () => { it('clamps percent_remaining to 0-100 range', () => {
const above = normalizeGhcpSnapshot({ const above = normalizeGhcpSnapshot({
entitlement: 100, entitlement: 100,
@@ -3,7 +3,7 @@
* *
* Tests the quota monitor lifecycle: * Tests the quota monitor lifecycle:
* - startQuotaMonitor / stopQuotaMonitor behavior * - startQuotaMonitor / stopQuotaMonitor behavior
* - No-op conditions for non-agy, manual mode, disabled config * - No-op conditions for unsupported providers, manual mode, disabled config
* - Idempotent stopQuotaMonitor * - Idempotent stopQuotaMonitor
*/ */
@@ -16,6 +16,7 @@ import {
stopQuotaMonitor, stopQuotaMonitor,
clearQuotaCache, clearQuotaCache,
} from '../../../src/cliproxy/quota-manager'; } from '../../../src/cliproxy/quota-manager';
import { __testExports as executorTestExports } from '../../../src/cliproxy/executor';
// Setup test isolation // Setup test isolation
let tmpDir: string; let tmpDir: string;
@@ -40,11 +41,43 @@ afterEach(() => {
}); });
describe('Runtime Quota Monitor', () => { describe('Runtime Quota Monitor', () => {
describe('resolveRuntimeQuotaMonitorProviders', () => {
it('should monitor the default provider for non-composite sessions', () => {
expect(executorTestExports.resolveRuntimeQuotaMonitorProviders('codex', [])).toEqual([
'codex',
]);
});
it('should monitor every unique managed provider in composite sessions', () => {
expect(
executorTestExports.resolveRuntimeQuotaMonitorProviders('claude', [
'codex',
'qwen',
'gemini',
'codex',
])
).toEqual(['codex', 'gemini']);
});
it('should skip unsupported non-composite providers', () => {
expect(executorTestExports.resolveRuntimeQuotaMonitorProviders('qwen', [])).toEqual([]);
});
});
describe('startQuotaMonitor', () => { describe('startQuotaMonitor', () => {
it('should accept non-agy provider without throwing', () => { it('should accept all quota-supported providers without throwing', () => {
// Non-agy providers should be silently ignored for (const provider of ['agy', 'claude', 'codex', 'gemini', 'ghcp'] as const) {
expect(() => {
startQuotaMonitor(provider, 'test@gmail.com');
stopQuotaMonitor();
}).not.toThrow();
}
});
it('should ignore unsupported providers without throwing', () => {
expect(() => { expect(() => {
startQuotaMonitor('gemini', 'test@gmail.com'); startQuotaMonitor('qwen', 'test@gmail.com');
stopQuotaMonitor();
}).not.toThrow(); }).not.toThrow();
}); });
@@ -75,6 +108,54 @@ describe('Runtime Quota Monitor', () => {
}).not.toThrow(); }).not.toThrow();
}); });
it('should start independent monitors for distinct managed provider accounts', () => {
const configDir = path.join(tmpDir, '.ccs', 'config');
fs.mkdirSync(configDir, { recursive: true });
fs.writeFileSync(
path.join(configDir, 'unified-config.json'),
JSON.stringify({
version: 2,
quota_management: {
mode: 'auto',
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 5,
},
},
})
);
const originalSetTimeout = globalThis.setTimeout;
const fakeTimer = {
unref: () => fakeTimer,
ref: () => fakeTimer,
hasRef: () => false,
refresh: () => fakeTimer,
} as unknown as ReturnType<typeof setTimeout>;
const observedDelays: Array<number | undefined> = [];
globalThis.setTimeout = ((handler: TimerHandler, timeout?: number) => {
observedDelays.push(timeout);
void handler;
return fakeTimer;
}) as typeof globalThis.setTimeout;
try {
startQuotaMonitor('codex', 'codex@example.com');
startQuotaMonitor('gemini', 'gemini@example.com');
startQuotaMonitor('codex', 'codex@example.com');
} finally {
stopQuotaMonitor();
globalThis.setTimeout = originalSetTimeout;
}
expect(observedDelays).toEqual([300_000, 300_000]);
});
it('should be no-op when config missing or no quota_management', () => { it('should be no-op when config missing or no quota_management', () => {
// No config file — should not throw // No config file — should not throw
expect(() => { expect(() => {
@@ -465,7 +465,7 @@ export function QuotaTooltipContent({ quota, resetTime }: QuotaTooltipContentPro
}, },
{ label: t('quotaTooltip.chat'), snapshot: quota.snapshots.chat }, { label: t('quotaTooltip.chat'), snapshot: quota.snapshots.chat },
{ label: t('quotaTooltip.completions'), snapshot: quota.snapshots.completions }, { label: t('quotaTooltip.completions'), snapshot: quota.snapshots.completions },
]; ].filter(({ snapshot }) => snapshot.reported !== false);
const effectiveResetTime = quota.quotaResetDate ?? resetTime; const effectiveResetTime = quota.quotaResetDate ?? resetTime;
const planLabel = formatPlanLabel(quota.planType); const planLabel = formatPlanLabel(quota.planType);
+2
View File
@@ -862,6 +862,8 @@ export interface GeminiCliQuotaResult {
/** GitHub Copilot quota snapshot */ /** GitHub Copilot quota snapshot */
export interface GhcpQuotaSnapshot { export interface GhcpQuotaSnapshot {
/** False when upstream omitted this quota category */
reported?: boolean;
/** Total quota allocation for this category */ /** Total quota allocation for this category */
entitlement: number; entitlement: number;
/** Remaining quota count */ /** Remaining quota count */
+4 -5
View File
@@ -694,11 +694,10 @@ export function getGeminiResetTime(buckets: GeminiCliBucket[]): string | null {
export function getMinGhcpQuota(snapshots: GhcpQuotaResult['snapshots']): number | null { export function getMinGhcpQuota(snapshots: GhcpQuotaResult['snapshots']): number | null {
if (!snapshots) return null; if (!snapshots) return null;
const percentages = [ const percentages = [snapshots.premiumInteractions, snapshots.chat, snapshots.completions]
snapshots.premiumInteractions.percentRemaining, .filter((snapshot) => snapshot.reported !== false)
snapshots.chat.percentRemaining, .map((snapshot) => (snapshot.unlimited ? 100 : snapshot.percentRemaining))
snapshots.completions.percentRemaining, .filter((p) => typeof p === 'number' && isFinite(p));
].filter((p) => typeof p === 'number' && isFinite(p));
if (percentages.length === 0) return null; if (percentages.length === 0) return null;
return Math.min(...percentages); return Math.min(...percentages);