diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index 33f018b5..2a212ea8 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -41,6 +41,12 @@ All major modularization work is complete. The codebase evolved from monolithic ### Recent Fixes +- **2026-05-22**: **#1337** CLIProxy quota preflight now reconciles the entire + active provider account pool before local proxy startup. Known-exhausted + non-default accounts move out of live `auth/` rotation when another healthy + account exists, preventing round-robin requests from stalling on already + exhausted accounts while preserving manual pause/resume and cooldown + auto-resume behavior. - **2026-05-22**: **#1332** `ccsx auth` profiles now link native Codex `agents/` and `skills/` resources alongside `config.toml`, and `ccsx ` repairs missing resource links at launch so relative agent role `config_file` diff --git a/docs/system-architecture/provider-flows.md b/docs/system-architecture/provider-flows.md index 3a6f8d41..98c2e49d 100644 --- a/docs/system-architecture/provider-flows.md +++ b/docs/system-architecture/provider-flows.md @@ -269,7 +269,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise Check exhaustion status | + +---> Pause exhausted non-default accounts when another healthy account exists + | +---> Select best account (not paused, not exhausted) | +---> Auto-failover to next account if current exhausted diff --git a/src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts b/src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts index 4ef4205e..32b59c5c 100644 --- a/src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts +++ b/src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts @@ -56,12 +56,12 @@ function writeRegistry(providers: Record): void { // Helper: write unified config function writeConfig(quotaConfig: unknown): void { - const configDir = path.join(tmpDir, '.ccs', 'config'); + const configDir = path.join(tmpDir, '.ccs'); fs.mkdirSync(configDir, { recursive: true }); fs.writeFileSync( - path.join(configDir, 'unified-config.json'), + path.join(configDir, 'config.yaml'), JSON.stringify({ - version: 2, + version: 13, quota_management: quotaConfig, }) ); @@ -576,6 +576,258 @@ describe('Quota Exhaustion Handlers', () => { ).toBe(true); }); + it('should self-pause exhausted non-default Codex accounts before CLIProxy rotation', async () => { + writeRegistry({ + codex: { + default: 'rotation-default@example.com', + accounts: { + 'rotation-default@example.com': { + email: 'rotation-default@example.com', + tokenFile: 'codex-rotation-default.json', + }, + 'rotation-exhausted@example.com': { + email: 'rotation-exhausted@example.com', + tokenFile: 'codex-rotation-exhausted.json', + }, + 'rotation-healthy@example.com': { + email: 'rotation-healthy@example.com', + tokenFile: 'codex-rotation-healthy.json', + }, + }, + }, + }); + + writeConfig({ + mode: 'auto', + auto: { + tier_priority: ['ultra', 'pro', 'free'], + exhaustion_threshold: 5, + cooldown_minutes: 10, + preflight_check: true, + }, + runtime_monitor: { + enabled: true, + normal_interval_seconds: 300, + critical_interval_seconds: 60, + warn_threshold: 20, + exhaustion_threshold: 5, + cooldown_minutes: 10, + }, + }); + + writeCodexAuth( + 'codex-rotation-default.json', + 'rotation-default@example.com', + 'rotation-default-token' + ); + writeCodexAuth( + 'codex-rotation-exhausted.json', + 'rotation-exhausted@example.com', + 'rotation-exhausted-token' + ); + writeCodexAuth( + 'codex-rotation-healthy.json', + 'rotation-healthy@example.com', + 'rotation-healthy-token' + ); + + global.fetch = mock((_url: string, options?: RequestInit) => { + const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? ''; + const usedPercent = accountHeader === 'chatgpt-rotation-exhausted@example.com' ? 100 : 10; + return Promise.resolve( + Response.json({ + plan_type: 'pro', + rate_limit: { + primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 }, + secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 }, + }, + }) + ); + }) as typeof fetch; + + const { preflightCheck } = await import('../../quota/quota-manager'); + const result = await preflightCheck('codex'); + const { getAccount, getDefaultAccount } = await import('../account-manager'); + + expect(result.accountId).toBe('rotation-default@example.com'); + expect(result.switchedFrom).toBeUndefined(); + expect(getDefaultAccount('codex')?.id).toBe('rotation-default@example.com'); + expect(getAccount('codex', 'rotation-default@example.com')?.paused).not.toBe(true); + expect(getAccount('codex', 'rotation-healthy@example.com')?.paused).not.toBe(true); + expect(getAccount('codex', 'rotation-exhausted@example.com')?.paused).toBe(true); + expect( + fs.existsSync( + path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-rotation-exhausted.json') + ) + ).toBe(true); + expect( + fs.existsSync( + path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-rotation-exhausted.json') + ) + ).toBe(false); + }); + + it('should durably pause non-default Codex accounts already on cooldown', async () => { + writeRegistry({ + codex: { + default: 'cooldown-default@example.com', + accounts: { + 'cooldown-default@example.com': { + email: 'cooldown-default@example.com', + tokenFile: 'codex-cooldown-default.json', + }, + 'cooldown-exhausted@example.com': { + email: 'cooldown-exhausted@example.com', + tokenFile: 'codex-cooldown-exhausted.json', + }, + }, + }, + }); + + writeConfig({ + mode: 'auto', + auto: { + tier_priority: ['ultra', 'pro', 'free'], + exhaustion_threshold: 5, + cooldown_minutes: 10, + preflight_check: true, + }, + runtime_monitor: { + enabled: true, + normal_interval_seconds: 300, + critical_interval_seconds: 60, + warn_threshold: 20, + exhaustion_threshold: 5, + cooldown_minutes: 10, + }, + }); + + writeCodexAuth( + 'codex-cooldown-default.json', + 'cooldown-default@example.com', + 'cooldown-default-token' + ); + writeCodexAuth( + 'codex-cooldown-exhausted.json', + 'cooldown-exhausted@example.com', + 'cooldown-exhausted-token' + ); + + const { applyCooldown, preflightCheck } = await import('../../quota/quota-manager'); + applyCooldown('codex', 'cooldown-exhausted@example.com', 10); + + global.fetch = mock((_url: string, options?: RequestInit) => { + const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? ''; + const usedPercent = accountHeader === 'chatgpt-cooldown-default@example.com' ? 10 : 100; + return Promise.resolve( + Response.json({ + plan_type: 'pro', + rate_limit: { + primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 }, + secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 }, + }, + }) + ); + }) as typeof fetch; + + const result = await preflightCheck('codex'); + const { getAccount } = await import('../account-manager'); + + expect(result.accountId).toBe('cooldown-default@example.com'); + expect(getAccount('codex', 'cooldown-exhausted@example.com')?.paused).toBe(true); + expect( + fs.existsSync( + path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-cooldown-exhausted.json') + ) + ).toBe(true); + expect( + fs.existsSync( + path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-cooldown-exhausted.json') + ) + ).toBe(false); + }); + + it('should reconcile exhausted rotation accounts before honoring forced default', async () => { + writeRegistry({ + codex: { + default: 'forced-default@example.com', + accounts: { + 'forced-default@example.com': { + email: 'forced-default@example.com', + tokenFile: 'codex-forced-default.json', + }, + 'forced-exhausted@example.com': { + email: 'forced-exhausted@example.com', + tokenFile: 'codex-forced-exhausted.json', + }, + 'forced-healthy@example.com': { + email: 'forced-healthy@example.com', + tokenFile: 'codex-forced-healthy.json', + }, + }, + }, + }); + + writeConfig({ + mode: 'auto', + auto: { + tier_priority: ['ultra', 'pro', 'free'], + exhaustion_threshold: 5, + cooldown_minutes: 10, + preflight_check: true, + }, + manual: { + forced_default: 'forced-default@example.com', + }, + runtime_monitor: { + enabled: true, + normal_interval_seconds: 300, + critical_interval_seconds: 60, + warn_threshold: 20, + exhaustion_threshold: 5, + cooldown_minutes: 10, + }, + }); + + writeCodexAuth('codex-forced-default.json', 'forced-default@example.com', 'forced-token'); + writeCodexAuth( + 'codex-forced-exhausted.json', + 'forced-exhausted@example.com', + 'forced-exhausted-token' + ); + writeCodexAuth('codex-forced-healthy.json', 'forced-healthy@example.com', 'healthy-token'); + + global.fetch = mock((_url: string, options?: RequestInit) => { + const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? ''; + const usedPercent = accountHeader === 'chatgpt-forced-exhausted@example.com' ? 100 : 10; + return Promise.resolve( + Response.json({ + plan_type: 'pro', + rate_limit: { + primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 }, + secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 }, + }, + }) + ); + }) as typeof fetch; + + const { preflightCheck } = await import('../../quota/quota-manager'); + const result = await preflightCheck('codex'); + const { getAccount, getDefaultAccount } = await import('../account-manager'); + + expect(result.accountId).toBe('forced-default@example.com'); + expect(result.reason).toBe('Forced default override'); + expect(getDefaultAccount('codex')?.id).toBe('forced-default@example.com'); + expect(getAccount('codex', 'forced-default@example.com')?.paused).not.toBe(true); + expect(getAccount('codex', 'forced-healthy@example.com')?.paused).not.toBe(true); + expect(getAccount('codex', 'forced-exhausted@example.com')?.paused).toBe(true); + expect( + fs.existsSync( + path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-forced-exhausted.json') + ) + ).toBe(true); + }); + it('should not durably self-pause during preflight when fallback quota is unknown', async () => { writeRegistry({ codex: { diff --git a/src/cliproxy/quota/quota-manager.ts b/src/cliproxy/quota/quota-manager.ts index 4493aeb6..8574b079 100644 --- a/src/cliproxy/quota/quota-manager.ts +++ b/src/cliproxy/quota/quota-manager.ts @@ -476,6 +476,75 @@ export async function findHealthyAccount( return selectable[0]; } +export async function reconcileExhaustedRotationAccounts( + provider: CLIProxyProvider +): Promise { + if (!isManagedQuotaProvider(provider)) { + return []; + } + + const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import( + '../accounts/account-safety' + ); + restoreExpiredQuotaPauses(); + + const config = loadOrCreateUnifiedConfig(); + const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5; + const cooldownMinutes = config.quota_management?.auto?.cooldown_minutes ?? 5; + + const activeAccounts = getProviderAccounts(provider).filter( + (account) => !isAccountPaused(provider, account.id) + ); + if (activeAccounts.length < 2) { + return []; + } + + const accountsWithQuota = await batchedMap( + activeAccounts, + async (account) => { + let quota = getCachedQuota(provider, account.id); + if (!quota) { + quota = await fetchQuotaWithDedup(provider, account.id); + } + + return { + account, + quotaPercent: calculateQuotaPercent(provider, quota), + }; + }, + 10 + ); + + const healthyAccountIds = new Set( + accountsWithQuota + .filter(({ account, quotaPercent }) => { + if (isOnCooldown(provider, account.id)) return false; + return quotaPercent !== null && quotaPercent >= threshold; + }) + .map(({ account }) => account.id) + ); + if (healthyAccountIds.size === 0) { + return []; + } + + const pausedAccountIds: string[] = []; + for (const { account, quotaPercent } of accountsWithQuota) { + const exhaustedByQuota = quotaPercent !== null && quotaPercent < threshold; + const exhaustedByCooldown = isOnCooldown(provider, account.id); + if (!exhaustedByQuota && !exhaustedByCooldown) continue; + + const hasOtherHealthyAccount = [...healthyAccountIds].some((id) => id !== account.id); + if (!hasOtherHealthyAccount) continue; + + applyCooldown(provider, account.id, cooldownMinutes); + if (pauseAccountForQuotaCooldown(provider, account.id, cooldownMinutes)) { + pausedAccountIds.push(account.id); + } + } + + return pausedAccountIds; +} + /** * Find and switch to a healthy account */ @@ -543,11 +612,13 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise a.id === forcedDefault); - if (forcedAccount) { + if (forcedAccount && !isAccountPaused(provider, forcedAccount.id)) { return { proceed: true, accountId: forcedAccount.id, reason: 'Forced default override' }; } }