From 478d64a50ab7d6aef8507efe92971641ad523b09 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 21 Apr 2026 12:34:07 -0400 Subject: [PATCH] fix(cliproxy): quarantine exhausted quota accounts --- docs/project-roadmap.md | 3 +- docs/system-architecture/provider-flows.md | 6 + src/cliproxy/account-safety.ts | 146 ++++++++++++++++++ src/cliproxy/quota-manager.ts | 36 ++++- .../account-safety-quota-exhaustion.test.ts | 139 +++++++++++++++++ 5 files changed, 326 insertions(+), 4 deletions(-) diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index a08866ec..0cb616b3 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -1,6 +1,6 @@ # CCS Project Roadmap -Last Updated: 2026-04-19 +Last Updated: 2026-04-21 Forward-looking roadmap documenting current priorities, GitHub issues, and future feature plans. @@ -41,6 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic ### Recent Fixes +- **2026-04-21**: CLIProxy quota failover now quarantines exhausted Claude and Antigravity accounts out of live rotation when a healthy fallback exists. CCS persists those quota-triggered pauses across launches, automatically resumes them after the configured cooldown window, and deliberately avoids auto-pausing the last available account so single-account setups still degrade gracefully instead of hard-locking themselves. - **2026-04-20**: **#1051** Browser automation now defaults safe-off for new installs and upgrades that do not already carry explicit browser settings. CCS changes both Claude Browser Attach and Codex Browser Tools to start with `enabled: false` and `policy: manual`, normalizes missing browser policies on upgrade back to `manual`, preserves explicit existing enablement, and updates status/help/docs so browser tooling is never implied to auto-expose unless users opt in. - **2026-04-19**: **#1051** Browser tooling now has an explicit exposure policy instead of only coarse enablement toggles. CCS adds `browser..policy` (`auto` or `manual`) for both Claude Browser Attach and Codex Browser Tools, exposes CLI-first policy controls through `ccs browser policy`, `ccs browser enable`, and `ccs browser disable`, and adds one-run launch overrides `--browser` and `--no-browser` so users can force browser tooling on or off without editing saved config. - **2026-04-19**: **#1049** Browser setup now has a real remediation path instead of status/doctor-only guidance. CCS adds `ccs browser setup` as the primary one-command flow for Claude Browser Attach, shortens managed browser-path output to home-relative display paths where appropriate, and updates browser readiness guidance to point users at setup first while keeping browser doctor read-only by default. diff --git a/docs/system-architecture/provider-flows.md b/docs/system-architecture/provider-flows.md index eb19fd6f..a1853c49 100644 --- a/docs/system-architecture/provider-flows.md +++ b/docs/system-architecture/provider-flows.md @@ -257,6 +257,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise Select best account (not paused, not exhausted) | +---> Auto-failover to next account if current exhausted + | + +---> Temporarily pause exhausted account when fallback exists + | - move token out of live auth discovery + | - persist cooldown expiry across launches + | - auto-resume only CCS-created quota pauses CLI Commands: ccs cliproxy pause --> Set isPaused=true in account-manager diff --git a/src/cliproxy/account-safety.ts b/src/cliproxy/account-safety.ts index f8f61e0c..d7b26607 100644 --- a/src/cliproxy/account-safety.ts +++ b/src/cliproxy/account-safety.ts @@ -37,10 +37,26 @@ interface AutoPausedFile { sessions: AutoPausedSession[]; } +interface QuotaPausedEntry { + provider: CLIProxyProvider; + accountId: string; + pausedAt: string; + until: number; + reason: 'quota_exhausted'; +} + +interface QuotaPausedFile { + entries: QuotaPausedEntry[]; +} + function getAutoPausedPath(): string { return path.join(getCcsDir(), 'cliproxy', 'auto-paused.json'); } +function getQuotaPausedPath(): string { + return path.join(getCcsDir(), 'cliproxy', 'quota-paused.json'); +} + function loadAutoPaused(): AutoPausedFile { try { const filePath = getAutoPausedPath(); @@ -69,6 +85,48 @@ function saveAutoPaused(data: AutoPausedFile): void { fs.writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 }); } +function loadQuotaPaused(): QuotaPausedFile { + try { + const filePath = getQuotaPausedPath(); + if (fs.existsSync(filePath)) { + const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as { + entries?: unknown; + }; + if (Array.isArray(data.entries)) { + return { + entries: data.entries.filter( + (entry): entry is QuotaPausedEntry => + typeof entry === 'object' && + entry !== null && + typeof (entry as QuotaPausedEntry).provider === 'string' && + typeof (entry as QuotaPausedEntry).accountId === 'string' && + typeof (entry as QuotaPausedEntry).pausedAt === 'string' && + Number.isFinite((entry as QuotaPausedEntry).until) + ), + }; + } + } + } catch { + // Corrupted or malformed file — start fresh + } + return { entries: [] }; +} + +function saveQuotaPaused(data: QuotaPausedFile): void { + const filePath = getQuotaPausedPath(); + if (data.entries.length === 0) { + try { + fs.unlinkSync(filePath); + } catch { + /* already gone */ + } + return; + } + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 }); +} + /** * Check if a process is alive. NOTE: PIDs can be recycled by the OS. * If a stale PID is reused by an unrelated process, cleanup is deferred until that process exits. @@ -325,6 +383,93 @@ export function cleanupStaleAutoPauses(): void { } } +/** + * Resume quota-paused accounts whose cooldown windows have expired. + * Auto-resume only applies to pauses created by CCS quota handling. + */ +export function restoreExpiredQuotaPauses(now = Date.now()): number { + const data = loadQuotaPaused(); + if (data.entries.length === 0) return 0; + + const keep: QuotaPausedEntry[] = []; + let resumed = 0; + const registry = loadAccountsRegistry(); + + for (const entry of data.entries) { + if (!Number.isFinite(entry.until) || entry.until > now) { + keep.push(entry); + continue; + } + + const account = registry.providers[entry.provider]?.accounts[entry.accountId]; + if (!account?.paused) { + continue; + } + + // Only auto-resume the exact pause CCS created for quota cooldown. + // Missing or changed pausedAt metadata is treated as a mismatch so we do + // not accidentally resume a manually paused account. + if (account.pausedAt !== entry.pausedAt) { + continue; + } + + if (resumeAccount(entry.provider, entry.accountId)) { + resumed += 1; + continue; + } + + // Resume failures are treated as transient I/O/state issues. Keep the + // quota-pause record so the next restore pass can retry instead of leaving + // the account paused forever without any cooldown metadata. + keep.push(entry); + } + + saveQuotaPaused({ entries: keep }); + return resumed; +} + +/** + * Temporarily remove an exhausted account from CLIProxy rotation for the + * configured cooldown window. Returns false when the account was already paused + * or could not be paused, so callers can fall back to in-memory cooldown only. + */ +export function pauseAccountForQuotaCooldown( + provider: CLIProxyProvider, + accountId: string, + cooldownMinutes: number, + now = Date.now() +): boolean { + const registryBefore = loadAccountsRegistry(); + const accountBefore = registryBefore.providers[provider]?.accounts[accountId]; + if (!accountBefore || accountBefore.paused) { + return false; + } + + if (!pauseAccount(provider, accountId)) { + return false; + } + + const registryAfter = loadAccountsRegistry(); + const pausedAt = registryAfter.providers[provider]?.accounts[accountId]?.pausedAt; + if (!pausedAt) { + return false; + } + + const data = loadQuotaPaused(); + data.entries = data.entries.filter( + (entry) => !(entry.provider === provider && entry.accountId === accountId) + ); + data.entries.push({ + provider, + accountId, + pausedAt, + until: now + cooldownMinutes * 60 * 1000, + reason: 'quota_exhausted', + }); + saveQuotaPaused(data); + return true; +} + /** * Enforce provider isolation by auto-pausing conflicting accounts in other providers. * Records paused accounts for crash recovery and session exit restore. @@ -553,6 +698,7 @@ export async function handleQuotaExhaustion( const alternative = await findHealthyAccount(provider, [accountId]); if (alternative) { + pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes); setDefaultAccount(provider, alternative.id); touchAccount(provider, alternative.id); writeQuotaExhausted(accountId, alternative.id, cooldownMinutes); diff --git a/src/cliproxy/quota-manager.ts b/src/cliproxy/quota-manager.ts index 31c46780..ce3b59a8 100644 --- a/src/cliproxy/quota-manager.ts +++ b/src/cliproxy/quota-manager.ts @@ -286,6 +286,9 @@ export async function findHealthyAccount( return null; } + const { restoreExpiredQuotaPauses } = await import('./account-safety'); + restoreExpiredQuotaPauses(); + const config = loadOrCreateUnifiedConfig(); const tierPriority = config.quota_management?.auto?.tier_priority ?? ['ultra', 'pro', 'free']; const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5; @@ -388,6 +391,11 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise; }> { + const { restoreExpiredQuotaPauses } = await import('./account-safety'); + restoreExpiredQuotaPauses(); + const accounts = getProviderAccounts(provider); const defaultAccount = getDefaultAccount(provider); diff --git a/tests/unit/cliproxy/account-safety-quota-exhaustion.test.ts b/tests/unit/cliproxy/account-safety-quota-exhaustion.test.ts index 4ad5145f..e688e009 100644 --- a/tests/unit/cliproxy/account-safety-quota-exhaustion.test.ts +++ b/tests/unit/cliproxy/account-safety-quota-exhaustion.test.ts @@ -17,6 +17,8 @@ import { handleQuotaExhaustion, writeQuotaWarning, maskEmail, + pauseAccountForQuotaCooldown, + restoreExpiredQuotaPauses, } from '../../../src/cliproxy/account-safety'; import { sanitizeEmail } from '../../../src/cliproxy/auth-utils'; @@ -84,6 +86,12 @@ function writeClaudeAuth(accountId: string, accessToken: string): void { ); } +function writeAuthToken(tokenFile: string, payload: Record): void { + const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth'); + fs.mkdirSync(authDir, { recursive: true }); + fs.writeFileSync(path.join(authDir, tokenFile), JSON.stringify(payload, null, 2)); +} + describe('Quota Exhaustion Handlers', () => { describe('writeQuotaWarning', () => { it('should write to stderr with box format', async () => { @@ -247,10 +255,13 @@ describe('Quota Exhaustion Handlers', () => { }); const result = await handleQuotaExhaustion('agy', 'only@gmail.com', 10); + const { getAccount } = await import('../../../src/cliproxy/account-manager'); // Should return gracefully with null switched expect(result.switchedTo).toBeNull(); expect(result.reason).toContain('no alternatives'); + expect(getAccount('agy', 'only@gmail.com')?.paused).not.toBe(true); + expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false); }); it('should switch Claude accounts when fallback quota endpoint returns 404', async () => { @@ -305,6 +316,18 @@ describe('Quota Exhaustion Handlers', () => { expect(result.switchedTo).toBe('fallback@example.com'); expect(getDefaultAccount('claude')?.id).toBe('fallback@example.com'); + expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true); + expect( + fs.existsSync( + path.join( + tmpDir, + '.ccs', + 'cliproxy', + 'auth-paused', + `claude-${sanitizeEmail('exhausted@example.com')}.json` + ) + ) + ).toBe(true); }); it('should write warning to stderr', async () => { @@ -389,5 +412,121 @@ describe('Quota Exhaustion Handlers', () => { expect(result).toBeDefined(); expect(result.switchedTo).toBeNull(); }); + + it('auto-resumes quota-paused accounts after cooldown expiry', async () => { + writeRegistry({ + agy: { + default: 'cooldown@gmail.com', + accounts: { + 'cooldown@gmail.com': { + email: 'cooldown@gmail.com', + tokenFile: 'agy-cooldown.json', + }, + }, + }, + }); + writeAuthToken('agy-cooldown.json', { + type: 'agy', + email: 'cooldown@gmail.com', + access_token: 'token', + }); + + const now = Date.now(); + expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true); + + const { getAccount } = await import('../../../src/cliproxy/account-manager'); + expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true); + + const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000); + + expect(resumed).toBe(1); + expect(getAccount('agy', 'cooldown@gmail.com')?.paused).not.toBe(true); + expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false); + }); + + it('does not auto-resume quota-paused accounts when pausedAt metadata is missing', async () => { + writeRegistry({ + agy: { + default: 'cooldown@gmail.com', + accounts: { + 'cooldown@gmail.com': { + email: 'cooldown@gmail.com', + tokenFile: 'agy-cooldown.json', + }, + }, + }, + }); + writeAuthToken('agy-cooldown.json', { + type: 'agy', + email: 'cooldown@gmail.com', + access_token: 'token', + }); + + const now = Date.now(); + expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true); + + const registryPath = path.join(tmpDir, '.ccs', 'cliproxy', 'accounts.json'); + const registry = JSON.parse(fs.readFileSync(registryPath, 'utf8')) as { + providers: { + agy: { + default: string; + accounts: Record; + }; + }; + }; + delete registry.providers.agy.accounts['cooldown@gmail.com']?.pausedAt; + fs.writeFileSync(registryPath, JSON.stringify(registry, null, 2)); + + const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000); + const { getAccount } = await import('../../../src/cliproxy/account-manager'); + + expect(resumed).toBe(0); + expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true); + expect( + fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'agy-cooldown.json')) + ).toBe(true); + expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false); + }); + + it('keeps quota-paused entries when auto-resume fails and retries later', async () => { + writeRegistry({ + agy: { + default: 'cooldown@gmail.com', + accounts: { + 'cooldown@gmail.com': { + email: 'cooldown@gmail.com', + tokenFile: 'agy-cooldown.json', + }, + }, + }, + }); + writeAuthToken('agy-cooldown.json', { + type: 'agy', + email: 'cooldown@gmail.com', + access_token: 'token', + }); + + const now = Date.now(); + expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true); + + fs.rmSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth'), { + recursive: true, + force: true, + }); + + const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000); + const { getAccount } = await import('../../../src/cliproxy/account-manager'); + const quotaPausedPath = path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'); + const quotaPaused = JSON.parse(fs.readFileSync(quotaPausedPath, 'utf8')) as { + entries?: Array<{ accountId?: string }>; + }; + + expect(resumed).toBe(0); + expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true); + expect( + fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'agy-cooldown.json')) + ).toBe(true); + expect(quotaPaused.entries?.map((entry) => entry.accountId)).toContain('cooldown@gmail.com'); + }); }); });