fix(cliproxy): quarantine exhausted quota accounts

docs/project-roadmap.md

@@ -1,6 +1,6 @@
 # CCS Project Roadmap
 
-Last Updated: 2026-04-19
+Last Updated: 2026-04-21
 
 Forward-looking roadmap documenting current priorities, GitHub issues, and future feature plans.
 
@@ -41,6 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic
 
 ### Recent Fixes
 
+- **2026-04-21**: CLIProxy quota failover now quarantines exhausted Claude and Antigravity accounts out of live rotation when a healthy fallback exists. CCS persists those quota-triggered pauses across launches, automatically resumes them after the configured cooldown window, and deliberately avoids auto-pausing the last available account so single-account setups still degrade gracefully instead of hard-locking themselves.
 - **2026-04-20**: **#1051** Browser automation now defaults safe-off for new installs and upgrades that do not already carry explicit browser settings. CCS changes both Claude Browser Attach and Codex Browser Tools to start with `enabled: false` and `policy: manual`, normalizes missing browser policies on upgrade back to `manual`, preserves explicit existing enablement, and updates status/help/docs so browser tooling is never implied to auto-expose unless users opt in.
 - **2026-04-19**: **#1051** Browser tooling now has an explicit exposure policy instead of only coarse enablement toggles. CCS adds `browser.<lane>.policy` (`auto` or `manual`) for both Claude Browser Attach and Codex Browser Tools, exposes CLI-first policy controls through `ccs browser policy`, `ccs browser enable`, and `ccs browser disable`, and adds one-run launch overrides `--browser` and `--no-browser` so users can force browser tooling on or off without editing saved config.
 - **2026-04-19**: **#1049** Browser setup now has a real remediation path instead of status/doctor-only guidance. CCS adds `ccs browser setup` as the primary one-command flow for Claude Browser Attach, shortens managed browser-path output to home-relative display paths where appropriate, and updates browser readiness guidance to point users at setup first while keeping browser doctor read-only by default.

docs/system-architecture/provider-flows.md

@@ -257,6 +257,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise<bool
 ### Overview
 
 Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account.
+When CCS detects exhaustion and a healthy fallback exists, it also temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires.
 
 ```
 +===========================================================================+
@@ -293,6 +294,11 @@ Hybrid quota management enables automatic detection of exhausted accounts and fa
         +---> Select best account (not paused, not exhausted)
         |
         +---> Auto-failover to next account if current exhausted
+        |
+        +---> Temporarily pause exhausted account when fallback exists
+        |       - move token out of live auth discovery
+        |       - persist cooldown expiry across launches
+        |       - auto-resume only CCS-created quota pauses
 
   CLI Commands:
     ccs cliproxy pause <account>   --> Set isPaused=true in account-manager

src/cliproxy/account-safety.ts

@@ -37,10 +37,26 @@ interface AutoPausedFile {
   sessions: AutoPausedSession[];
 }
 
+interface QuotaPausedEntry {
+  provider: CLIProxyProvider;
+  accountId: string;
+  pausedAt: string;
+  until: number;
+  reason: 'quota_exhausted';
+}
+
+interface QuotaPausedFile {
+  entries: QuotaPausedEntry[];
+}
+
 function getAutoPausedPath(): string {
   return path.join(getCcsDir(), 'cliproxy', 'auto-paused.json');
 }
 
+function getQuotaPausedPath(): string {
+  return path.join(getCcsDir(), 'cliproxy', 'quota-paused.json');
+}
+
 function loadAutoPaused(): AutoPausedFile {
   try {
     const filePath = getAutoPausedPath();
@@ -69,6 +85,48 @@ function saveAutoPaused(data: AutoPausedFile): void {
   fs.writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 });
 }
 
+function loadQuotaPaused(): QuotaPausedFile {
+  try {
+    const filePath = getQuotaPausedPath();
+    if (fs.existsSync(filePath)) {
+      const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as {
+        entries?: unknown;
+      };
+      if (Array.isArray(data.entries)) {
+        return {
+          entries: data.entries.filter(
+            (entry): entry is QuotaPausedEntry =>
+              typeof entry === 'object' &&
+              entry !== null &&
+              typeof (entry as QuotaPausedEntry).provider === 'string' &&
+              typeof (entry as QuotaPausedEntry).accountId === 'string' &&
+              typeof (entry as QuotaPausedEntry).pausedAt === 'string' &&
+              Number.isFinite((entry as QuotaPausedEntry).until)
+          ),
+        };
+      }
+    }
+  } catch {
+    // Corrupted or malformed file — start fresh
+  }
+  return { entries: [] };
+}
+
+function saveQuotaPaused(data: QuotaPausedFile): void {
+  const filePath = getQuotaPausedPath();
+  if (data.entries.length === 0) {
+    try {
+      fs.unlinkSync(filePath);
+    } catch {
+      /* already gone */
+    }
+    return;
+  }
+  const dir = path.dirname(filePath);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+  fs.writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 });
+}
+
 /**
  * Check if a process is alive. NOTE: PIDs can be recycled by the OS.
  * If a stale PID is reused by an unrelated process, cleanup is deferred until that process exits.
@@ -325,6 +383,93 @@ export function cleanupStaleAutoPauses(): void {
   }
 }
 
+/**
+ * Resume quota-paused accounts whose cooldown windows have expired.
+ * Auto-resume only applies to pauses created by CCS quota handling.
+ */
+export function restoreExpiredQuotaPauses(now = Date.now()): number {
+  const data = loadQuotaPaused();
+  if (data.entries.length === 0) return 0;
+
+  const keep: QuotaPausedEntry[] = [];
+  let resumed = 0;
+  const registry = loadAccountsRegistry();
+
+  for (const entry of data.entries) {
+    if (!Number.isFinite(entry.until) || entry.until > now) {
+      keep.push(entry);
+      continue;
+    }
+
+    const account = registry.providers[entry.provider]?.accounts[entry.accountId];
+    if (!account?.paused) {
+      continue;
+    }
+
+    // Only auto-resume the exact pause CCS created for quota cooldown.
+    // Missing or changed pausedAt metadata is treated as a mismatch so we do
+    // not accidentally resume a manually paused account.
+    if (account.pausedAt !== entry.pausedAt) {
+      continue;
+    }
+
+    if (resumeAccount(entry.provider, entry.accountId)) {
+      resumed += 1;
+      continue;
+    }
+
+    // Resume failures are treated as transient I/O/state issues. Keep the
+    // quota-pause record so the next restore pass can retry instead of leaving
+    // the account paused forever without any cooldown metadata.
+    keep.push(entry);
+  }
+
+  saveQuotaPaused({ entries: keep });
+  return resumed;
+}
+
+/**
+ * Temporarily remove an exhausted account from CLIProxy rotation for the
+ * configured cooldown window. Returns false when the account was already paused
+ * or could not be paused, so callers can fall back to in-memory cooldown only.
+ */
+export function pauseAccountForQuotaCooldown(
+  provider: CLIProxyProvider,
+  accountId: string,
+  cooldownMinutes: number,
+  now = Date.now()
+): boolean {
+  const registryBefore = loadAccountsRegistry();
+  const accountBefore = registryBefore.providers[provider]?.accounts[accountId];
+  if (!accountBefore || accountBefore.paused) {
+    return false;
+  }
+
+  if (!pauseAccount(provider, accountId)) {
+    return false;
+  }
+
+  const registryAfter = loadAccountsRegistry();
+  const pausedAt = registryAfter.providers[provider]?.accounts[accountId]?.pausedAt;
+  if (!pausedAt) {
+    return false;
+  }
+
+  const data = loadQuotaPaused();
+  data.entries = data.entries.filter(
+    (entry) => !(entry.provider === provider && entry.accountId === accountId)
+  );
+  data.entries.push({
+    provider,
+    accountId,
+    pausedAt,
+    until: now + cooldownMinutes * 60 * 1000,
+    reason: 'quota_exhausted',
+  });
+  saveQuotaPaused(data);
+  return true;
+}
+
 /**
  * Enforce provider isolation by auto-pausing conflicting accounts in other providers.
  * Records paused accounts for crash recovery and session exit restore.
@@ -553,6 +698,7 @@ export async function handleQuotaExhaustion(
   const alternative = await findHealthyAccount(provider, [accountId]);
 
   if (alternative) {
+    pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes);
     setDefaultAccount(provider, alternative.id);
     touchAccount(provider, alternative.id);
     writeQuotaExhausted(accountId, alternative.id, cooldownMinutes);

src/cliproxy/quota-manager.ts

@@ -286,6 +286,9 @@ export async function findHealthyAccount(
     return null;
   }
 
+  const { restoreExpiredQuotaPauses } = await import('./account-safety');
+  restoreExpiredQuotaPauses();
+
   const config = loadOrCreateUnifiedConfig();
   const tierPriority = config.quota_management?.auto?.tier_priority ?? ['ultra', 'pro', 'free'];
   const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5;
@@ -388,6 +391,11 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
     return { proceed: true, accountId: defaultAccount?.id || '' };
   }
 
+  const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import(
+    './account-safety'
+  );
+  restoreExpiredQuotaPauses();
+
   const config = loadOrCreateUnifiedConfig();
   const quotaConfig = config.quota_management;
 
@@ -442,13 +450,32 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
   const threshold = quotaConfig.auto?.exhaustion_threshold ?? 5;
 
   if (avgQuota < threshold) {
-    // Apply cooldown to exhausted account
     applyCooldown(provider, defaultAccount.id, quotaConfig.auto?.cooldown_minutes ?? 5);
-    return await findAndSwitch(
+
+    const alternative = await findHealthyAccount(provider, [defaultAccount.id]);
+    if (!alternative) {
+      return {
+        proceed: true,
+        accountId: defaultAccount.id,
+        reason: `Quota exhausted (${avgQuota.toFixed(1)}%), no alternatives available`,
+        quotaPercent,
+      };
+    }
+
+    pauseAccountForQuotaCooldown(
       provider,
       defaultAccount.id,
-      `Quota exhausted (${avgQuota.toFixed(1)}%)`
+      quotaConfig.auto?.cooldown_minutes ?? 5
     );
+    setDefaultAccount(provider, alternative.id);
+    touchAccount(provider, alternative.id);
+    return {
+      proceed: true,
+      accountId: alternative.id,
+      switchedFrom: defaultAccount.id,
+      reason: `Quota exhausted (${avgQuota.toFixed(1)}%)`,
+      quotaPercent: alternative.lastQuota,
+    };
   }
 
   return {
@@ -471,6 +498,9 @@ export async function getQuotaStatus(provider: CLIProxyProvider): Promise<{
     isDefault: boolean;
   }>;
 }> {
+  const { restoreExpiredQuotaPauses } = await import('./account-safety');
+  restoreExpiredQuotaPauses();
+
   const accounts = getProviderAccounts(provider);
   const defaultAccount = getDefaultAccount(provider);
 

tests/unit/cliproxy/account-safety-quota-exhaustion.test.ts

@@ -17,6 +17,8 @@ import {
   handleQuotaExhaustion,
   writeQuotaWarning,
   maskEmail,
+  pauseAccountForQuotaCooldown,
+  restoreExpiredQuotaPauses,
 } from '../../../src/cliproxy/account-safety';
 import { sanitizeEmail } from '../../../src/cliproxy/auth-utils';
 
@@ -84,6 +86,12 @@ function writeClaudeAuth(accountId: string, accessToken: string): void {
   );
 }
 
+function writeAuthToken(tokenFile: string, payload: Record<string, unknown>): void {
+  const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth');
+  fs.mkdirSync(authDir, { recursive: true });
+  fs.writeFileSync(path.join(authDir, tokenFile), JSON.stringify(payload, null, 2));
+}
+
 describe('Quota Exhaustion Handlers', () => {
   describe('writeQuotaWarning', () => {
     it('should write to stderr with box format', async () => {
@@ -247,10 +255,13 @@ describe('Quota Exhaustion Handlers', () => {
       });
 
       const result = await handleQuotaExhaustion('agy', 'only@gmail.com', 10);
+      const { getAccount } = await import('../../../src/cliproxy/account-manager');
 
       // Should return gracefully with null switched
       expect(result.switchedTo).toBeNull();
       expect(result.reason).toContain('no alternatives');
+      expect(getAccount('agy', 'only@gmail.com')?.paused).not.toBe(true);
+      expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
     });
 
     it('should switch Claude accounts when fallback quota endpoint returns 404', async () => {
@@ -305,6 +316,18 @@ describe('Quota Exhaustion Handlers', () => {
 
       expect(result.switchedTo).toBe('fallback@example.com');
       expect(getDefaultAccount('claude')?.id).toBe('fallback@example.com');
+      expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(
+            tmpDir,
+            '.ccs',
+            'cliproxy',
+            'auth-paused',
+            `claude-${sanitizeEmail('exhausted@example.com')}.json`
+          )
+        )
+      ).toBe(true);
     });
 
     it('should write warning to stderr', async () => {
@@ -389,5 +412,121 @@ describe('Quota Exhaustion Handlers', () => {
       expect(result).toBeDefined();
       expect(result.switchedTo).toBeNull();
     });
+
+    it('auto-resumes quota-paused accounts after cooldown expiry', async () => {
+      writeRegistry({
+        agy: {
+          default: 'cooldown@gmail.com',
+          accounts: {
+            'cooldown@gmail.com': {
+              email: 'cooldown@gmail.com',
+              tokenFile: 'agy-cooldown.json',
+            },
+          },
+        },
+      });
+      writeAuthToken('agy-cooldown.json', {
+        type: 'agy',
+        email: 'cooldown@gmail.com',
+        access_token: 'token',
+      });
+
+      const now = Date.now();
+      expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true);
+
+      const { getAccount } = await import('../../../src/cliproxy/account-manager');
+      expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true);
+
+      const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000);
+
+      expect(resumed).toBe(1);
+      expect(getAccount('agy', 'cooldown@gmail.com')?.paused).not.toBe(true);
+      expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
+    });
+
+    it('does not auto-resume quota-paused accounts when pausedAt metadata is missing', async () => {
+      writeRegistry({
+        agy: {
+          default: 'cooldown@gmail.com',
+          accounts: {
+            'cooldown@gmail.com': {
+              email: 'cooldown@gmail.com',
+              tokenFile: 'agy-cooldown.json',
+            },
+          },
+        },
+      });
+      writeAuthToken('agy-cooldown.json', {
+        type: 'agy',
+        email: 'cooldown@gmail.com',
+        access_token: 'token',
+      });
+
+      const now = Date.now();
+      expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true);
+
+      const registryPath = path.join(tmpDir, '.ccs', 'cliproxy', 'accounts.json');
+      const registry = JSON.parse(fs.readFileSync(registryPath, 'utf8')) as {
+        providers: {
+          agy: {
+            default: string;
+            accounts: Record<string, { paused?: boolean; pausedAt?: string }>;
+          };
+        };
+      };
+      delete registry.providers.agy.accounts['cooldown@gmail.com']?.pausedAt;
+      fs.writeFileSync(registryPath, JSON.stringify(registry, null, 2));
+
+      const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000);
+      const { getAccount } = await import('../../../src/cliproxy/account-manager');
+
+      expect(resumed).toBe(0);
+      expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true);
+      expect(
+        fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'agy-cooldown.json'))
+      ).toBe(true);
+      expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
+    });
+
+    it('keeps quota-paused entries when auto-resume fails and retries later', async () => {
+      writeRegistry({
+        agy: {
+          default: 'cooldown@gmail.com',
+          accounts: {
+            'cooldown@gmail.com': {
+              email: 'cooldown@gmail.com',
+              tokenFile: 'agy-cooldown.json',
+            },
+          },
+        },
+      });
+      writeAuthToken('agy-cooldown.json', {
+        type: 'agy',
+        email: 'cooldown@gmail.com',
+        access_token: 'token',
+      });
+
+      const now = Date.now();
+      expect(pauseAccountForQuotaCooldown('agy', 'cooldown@gmail.com', 5, now)).toBe(true);
+
+      fs.rmSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth'), {
+        recursive: true,
+        force: true,
+      });
+
+      const resumed = restoreExpiredQuotaPauses(now + 6 * 60 * 1000);
+      const { getAccount } = await import('../../../src/cliproxy/account-manager');
+      const quotaPausedPath = path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json');
+      const quotaPaused = JSON.parse(fs.readFileSync(quotaPausedPath, 'utf8')) as {
+        entries?: Array<{ accountId?: string }>;
+      };
+
+      expect(resumed).toBe(0);
+      expect(getAccount('agy', 'cooldown@gmail.com')?.paused).toBe(true);
+      expect(
+        fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'agy-cooldown.json'))
+      ).toBe(true);
+      expect(quotaPaused.entries?.map((entry) => entry.accountId)).toContain('cooldown@gmail.com');
+    });
   });
 });