fix: pause exhausted CLIProxy rotation accounts

Refs #1337

docs/project-roadmap.md

@@ -41,6 +41,12 @@ All major modularization work is complete. The codebase evolved from monolithic
 
 ### Recent Fixes
 
+- **2026-05-22**: **#1337** CLIProxy quota preflight now reconciles the entire
+  active provider account pool before local proxy startup. Known-exhausted
+  non-default accounts move out of live `auth/` rotation when another healthy
+  account exists, preventing round-robin requests from stalling on already
+  exhausted accounts while preserving manual pause/resume and cooldown
+  auto-resume behavior.
 - **2026-05-22**: **#1332** `ccsx auth` profiles now link native Codex
   `agents/` and `skills/` resources alongside `config.toml`, and `ccsx <profile>`
   repairs missing resource links at launch so relative agent role `config_file`

docs/system-architecture/provider-flows.md

@@ -269,7 +269,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise<bool
 ### Overview
 
 Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account.
-When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
+Before local CLIProxy startup, CCS reconciles the whole active account pool for the provider, not only the default account. When CCS detects any quota-exhausted account and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
 
 ```
 +===========================================================================+
@@ -305,6 +305,8 @@ When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses
         |           |
         |           +---> Check exhaustion status
         |
+        +---> Pause exhausted non-default accounts when another healthy account exists
+        |
         +---> Select best account (not paused, not exhausted)
         |
         +---> Auto-failover to next account if current exhausted

src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts

@@ -56,12 +56,12 @@ function writeRegistry(providers: Record<string, unknown>): void {
 
 // Helper: write unified config
 function writeConfig(quotaConfig: unknown): void {
-  const configDir = path.join(tmpDir, '.ccs', 'config');
+  const configDir = path.join(tmpDir, '.ccs');
   fs.mkdirSync(configDir, { recursive: true });
   fs.writeFileSync(
-    path.join(configDir, 'unified-config.json'),
+    path.join(configDir, 'config.yaml'),
     JSON.stringify({
-      version: 2,
+      version: 13,
       quota_management: quotaConfig,
     })
   );
@@ -576,6 +576,258 @@ describe('Quota Exhaustion Handlers', () => {
       ).toBe(true);
     });
 
+    it('should self-pause exhausted non-default Codex accounts before CLIProxy rotation', async () => {
+      writeRegistry({
+        codex: {
+          default: 'rotation-default@example.com',
+          accounts: {
+            'rotation-default@example.com': {
+              email: 'rotation-default@example.com',
+              tokenFile: 'codex-rotation-default.json',
+            },
+            'rotation-exhausted@example.com': {
+              email: 'rotation-exhausted@example.com',
+              tokenFile: 'codex-rotation-exhausted.json',
+            },
+            'rotation-healthy@example.com': {
+              email: 'rotation-healthy@example.com',
+              tokenFile: 'codex-rotation-healthy.json',
+            },
+          },
+        },
+      });
+
+      writeConfig({
+        mode: 'auto',
+        auto: {
+          tier_priority: ['ultra', 'pro', 'free'],
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+          preflight_check: true,
+        },
+        runtime_monitor: {
+          enabled: true,
+          normal_interval_seconds: 300,
+          critical_interval_seconds: 60,
+          warn_threshold: 20,
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+        },
+      });
+
+      writeCodexAuth(
+        'codex-rotation-default.json',
+        'rotation-default@example.com',
+        'rotation-default-token'
+      );
+      writeCodexAuth(
+        'codex-rotation-exhausted.json',
+        'rotation-exhausted@example.com',
+        'rotation-exhausted-token'
+      );
+      writeCodexAuth(
+        'codex-rotation-healthy.json',
+        'rotation-healthy@example.com',
+        'rotation-healthy-token'
+      );
+
+      global.fetch = mock((_url: string, options?: RequestInit) => {
+        const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
+        const usedPercent = accountHeader === 'chatgpt-rotation-exhausted@example.com' ? 100 : 10;
+        return Promise.resolve(
+          Response.json({
+            plan_type: 'pro',
+            rate_limit: {
+              primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
+              secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
+            },
+          })
+        );
+      }) as typeof fetch;
+
+      const { preflightCheck } = await import('../../quota/quota-manager');
+      const result = await preflightCheck('codex');
+      const { getAccount, getDefaultAccount } = await import('../account-manager');
+
+      expect(result.accountId).toBe('rotation-default@example.com');
+      expect(result.switchedFrom).toBeUndefined();
+      expect(getDefaultAccount('codex')?.id).toBe('rotation-default@example.com');
+      expect(getAccount('codex', 'rotation-default@example.com')?.paused).not.toBe(true);
+      expect(getAccount('codex', 'rotation-healthy@example.com')?.paused).not.toBe(true);
+      expect(getAccount('codex', 'rotation-exhausted@example.com')?.paused).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-rotation-exhausted.json')
+        )
+      ).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-rotation-exhausted.json')
+        )
+      ).toBe(false);
+    });
+
+    it('should durably pause non-default Codex accounts already on cooldown', async () => {
+      writeRegistry({
+        codex: {
+          default: 'cooldown-default@example.com',
+          accounts: {
+            'cooldown-default@example.com': {
+              email: 'cooldown-default@example.com',
+              tokenFile: 'codex-cooldown-default.json',
+            },
+            'cooldown-exhausted@example.com': {
+              email: 'cooldown-exhausted@example.com',
+              tokenFile: 'codex-cooldown-exhausted.json',
+            },
+          },
+        },
+      });
+
+      writeConfig({
+        mode: 'auto',
+        auto: {
+          tier_priority: ['ultra', 'pro', 'free'],
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+          preflight_check: true,
+        },
+        runtime_monitor: {
+          enabled: true,
+          normal_interval_seconds: 300,
+          critical_interval_seconds: 60,
+          warn_threshold: 20,
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+        },
+      });
+
+      writeCodexAuth(
+        'codex-cooldown-default.json',
+        'cooldown-default@example.com',
+        'cooldown-default-token'
+      );
+      writeCodexAuth(
+        'codex-cooldown-exhausted.json',
+        'cooldown-exhausted@example.com',
+        'cooldown-exhausted-token'
+      );
+
+      const { applyCooldown, preflightCheck } = await import('../../quota/quota-manager');
+      applyCooldown('codex', 'cooldown-exhausted@example.com', 10);
+
+      global.fetch = mock((_url: string, options?: RequestInit) => {
+        const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
+        const usedPercent = accountHeader === 'chatgpt-cooldown-default@example.com' ? 10 : 100;
+        return Promise.resolve(
+          Response.json({
+            plan_type: 'pro',
+            rate_limit: {
+              primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
+              secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
+            },
+          })
+        );
+      }) as typeof fetch;
+
+      const result = await preflightCheck('codex');
+      const { getAccount } = await import('../account-manager');
+
+      expect(result.accountId).toBe('cooldown-default@example.com');
+      expect(getAccount('codex', 'cooldown-exhausted@example.com')?.paused).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-cooldown-exhausted.json')
+        )
+      ).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-cooldown-exhausted.json')
+        )
+      ).toBe(false);
+    });
+
+    it('should reconcile exhausted rotation accounts before honoring forced default', async () => {
+      writeRegistry({
+        codex: {
+          default: 'forced-default@example.com',
+          accounts: {
+            'forced-default@example.com': {
+              email: 'forced-default@example.com',
+              tokenFile: 'codex-forced-default.json',
+            },
+            'forced-exhausted@example.com': {
+              email: 'forced-exhausted@example.com',
+              tokenFile: 'codex-forced-exhausted.json',
+            },
+            'forced-healthy@example.com': {
+              email: 'forced-healthy@example.com',
+              tokenFile: 'codex-forced-healthy.json',
+            },
+          },
+        },
+      });
+
+      writeConfig({
+        mode: 'auto',
+        auto: {
+          tier_priority: ['ultra', 'pro', 'free'],
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+          preflight_check: true,
+        },
+        manual: {
+          forced_default: 'forced-default@example.com',
+        },
+        runtime_monitor: {
+          enabled: true,
+          normal_interval_seconds: 300,
+          critical_interval_seconds: 60,
+          warn_threshold: 20,
+          exhaustion_threshold: 5,
+          cooldown_minutes: 10,
+        },
+      });
+
+      writeCodexAuth('codex-forced-default.json', 'forced-default@example.com', 'forced-token');
+      writeCodexAuth(
+        'codex-forced-exhausted.json',
+        'forced-exhausted@example.com',
+        'forced-exhausted-token'
+      );
+      writeCodexAuth('codex-forced-healthy.json', 'forced-healthy@example.com', 'healthy-token');
+
+      global.fetch = mock((_url: string, options?: RequestInit) => {
+        const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
+        const usedPercent = accountHeader === 'chatgpt-forced-exhausted@example.com' ? 100 : 10;
+        return Promise.resolve(
+          Response.json({
+            plan_type: 'pro',
+            rate_limit: {
+              primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
+              secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
+            },
+          })
+        );
+      }) as typeof fetch;
+
+      const { preflightCheck } = await import('../../quota/quota-manager');
+      const result = await preflightCheck('codex');
+      const { getAccount, getDefaultAccount } = await import('../account-manager');
+
+      expect(result.accountId).toBe('forced-default@example.com');
+      expect(result.reason).toBe('Forced default override');
+      expect(getDefaultAccount('codex')?.id).toBe('forced-default@example.com');
+      expect(getAccount('codex', 'forced-default@example.com')?.paused).not.toBe(true);
+      expect(getAccount('codex', 'forced-healthy@example.com')?.paused).not.toBe(true);
+      expect(getAccount('codex', 'forced-exhausted@example.com')?.paused).toBe(true);
+      expect(
+        fs.existsSync(
+          path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-forced-exhausted.json')
+        )
+      ).toBe(true);
+    });
+
     it('should not durably self-pause during preflight when fallback quota is unknown', async () => {
       writeRegistry({
         codex: {

src/cliproxy/quota/quota-manager.ts

@@ -476,6 +476,75 @@ export async function findHealthyAccount(
   return selectable[0];
 }
 
+export async function reconcileExhaustedRotationAccounts(
+  provider: CLIProxyProvider
+): Promise<string[]> {
+  if (!isManagedQuotaProvider(provider)) {
+    return [];
+  }
+
+  const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import(
+    '../accounts/account-safety'
+  );
+  restoreExpiredQuotaPauses();
+
+  const config = loadOrCreateUnifiedConfig();
+  const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5;
+  const cooldownMinutes = config.quota_management?.auto?.cooldown_minutes ?? 5;
+
+  const activeAccounts = getProviderAccounts(provider).filter(
+    (account) => !isAccountPaused(provider, account.id)
+  );
+  if (activeAccounts.length < 2) {
+    return [];
+  }
+
+  const accountsWithQuota = await batchedMap(
+    activeAccounts,
+    async (account) => {
+      let quota = getCachedQuota(provider, account.id);
+      if (!quota) {
+        quota = await fetchQuotaWithDedup(provider, account.id);
+      }
+
+      return {
+        account,
+        quotaPercent: calculateQuotaPercent(provider, quota),
+      };
+    },
+    10
+  );
+
+  const healthyAccountIds = new Set(
+    accountsWithQuota
+      .filter(({ account, quotaPercent }) => {
+        if (isOnCooldown(provider, account.id)) return false;
+        return quotaPercent !== null && quotaPercent >= threshold;
+      })
+      .map(({ account }) => account.id)
+  );
+  if (healthyAccountIds.size === 0) {
+    return [];
+  }
+
+  const pausedAccountIds: string[] = [];
+  for (const { account, quotaPercent } of accountsWithQuota) {
+    const exhaustedByQuota = quotaPercent !== null && quotaPercent < threshold;
+    const exhaustedByCooldown = isOnCooldown(provider, account.id);
+    if (!exhaustedByQuota && !exhaustedByCooldown) continue;
+
+    const hasOtherHealthyAccount = [...healthyAccountIds].some((id) => id !== account.id);
+    if (!hasOtherHealthyAccount) continue;
+
+    applyCooldown(provider, account.id, cooldownMinutes);
+    if (pauseAccountForQuotaCooldown(provider, account.id, cooldownMinutes)) {
+      pausedAccountIds.push(account.id);
+    }
+  }
+
+  return pausedAccountIds;
+}
+
 /**
  * Find and switch to a healthy account
  */
@@ -543,11 +612,13 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
     return { proceed: false, accountId: '', reason: 'No accounts configured' };
   }
 
+  await reconcileExhaustedRotationAccounts(provider);
+
   // Check forced_default override (manual mode)
   const forcedDefault = quotaConfig.manual?.forced_default;
   if (forcedDefault) {
     const forcedAccount = getProviderAccounts(provider).find((a) => a.id === forcedDefault);
-    if (forcedAccount) {
+    if (forcedAccount && !isAccountPaused(provider, forcedAccount.id)) {
       return { proceed: true, accountId: forcedAccount.id, reason: 'Forced default override' };
     }
   }