tiennm99/ccs
1
0
Fork 0
mirror of https://github.com/tiennm99/ccs.git synced 2026-07-15 16:19:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: pause exhausted CLIProxy rotation accounts

Browse Source 
Refs #1337
This commit is contained in:
Kai (Tam Nhu) Tran
2026-05-23 00:08:53 -04:00
committed by GitHub
parent 9e5729d3f4
commit 923bfee6fa
4 changed files with 336 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/project-roadmap.md
+6
View File
@@ -41,6 +41,12 @@ All major modularization work is complete. The codebase evolved from monolithic
### Recent Fixes
- **2026-05-22**: **#1337** CLIProxy quota preflight now reconciles the entire
active provider account pool before local proxy startup. Known-exhausted
non-default accounts move out of live `auth/` rotation when another healthy
account exists, preventing round-robin requests from stalling on already
exhausted accounts while preserving manual pause/resume and cooldown
auto-resume behavior.
- **2026-05-22**: **#1332** `ccsx auth` profiles now link native Codex
`agents/` and `skills/` resources alongside `config.toml`, and `ccsx <profile>`
repairs missing resource links at launch so relative agent role `config_file`
docs/system-architecture/provider-flows.md
+3 -1
View File
@@ -269,7 +269,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise<bool
### Overview
Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account.
When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
Before local CLIProxy startup, CCS reconciles the whole active account pool for the provider, not only the default account. When CCS detects any quota-exhausted account and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
```
+===========================================================================+
@@ -305,6 +305,8 @@ When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses
| |
| +---> Check exhaustion status
|
+---> Pause exhausted non-default accounts when another healthy account exists
|
+---> Select best account (not paused, not exhausted)
|
+---> Auto-failover to next account if current exhausted
src/cliproxy/accounts/__tests__/account-safety-quota-exhaustion.test.ts
+255 -3
View File
@@ -56,12 +56,12 @@ function writeRegistry(providers: Record<string, unknown>): void {
// Helper: write unified config
function writeConfig(quotaConfig: unknown): void {
const configDir = path.join(tmpDir, '.ccs', 'config');
const configDir = path.join(tmpDir, '.ccs');
fs.mkdirSync(configDir, { recursive: true });
fs.writeFileSync(
path.join(configDir, 'unified-config.json'),
path.join(configDir, 'config.yaml'),
JSON.stringify({
version: 2,
version: 13,
quota_management: quotaConfig,
})
);
@@ -576,6 +576,258 @@ describe('Quota Exhaustion Handlers', () => {
).toBe(true);
});
it('should self-pause exhausted non-default Codex accounts before CLIProxy rotation', async () => {
writeRegistry({
codex: {
default: 'rotation-default@example.com',
accounts: {
'rotation-default@example.com': {
email: 'rotation-default@example.com',
tokenFile: 'codex-rotation-default.json',
},
'rotation-exhausted@example.com': {
email: 'rotation-exhausted@example.com',
tokenFile: 'codex-rotation-exhausted.json',
},
'rotation-healthy@example.com': {
email: 'rotation-healthy@example.com',
tokenFile: 'codex-rotation-healthy.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth(
'codex-rotation-default.json',
'rotation-default@example.com',
'rotation-default-token'
);
writeCodexAuth(
'codex-rotation-exhausted.json',
'rotation-exhausted@example.com',
'rotation-exhausted-token'
);
writeCodexAuth(
'codex-rotation-healthy.json',
'rotation-healthy@example.com',
'rotation-healthy-token'
);
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
const usedPercent = accountHeader === 'chatgpt-rotation-exhausted@example.com' ? 100 : 10;
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const { preflightCheck } = await import('../../quota/quota-manager');
const result = await preflightCheck('codex');
const { getAccount, getDefaultAccount } = await import('../account-manager');
expect(result.accountId).toBe('rotation-default@example.com');
expect(result.switchedFrom).toBeUndefined();
expect(getDefaultAccount('codex')?.id).toBe('rotation-default@example.com');
expect(getAccount('codex', 'rotation-default@example.com')?.paused).not.toBe(true);
expect(getAccount('codex', 'rotation-healthy@example.com')?.paused).not.toBe(true);
expect(getAccount('codex', 'rotation-exhausted@example.com')?.paused).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-rotation-exhausted.json')
)
).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-rotation-exhausted.json')
)
).toBe(false);
});
it('should durably pause non-default Codex accounts already on cooldown', async () => {
writeRegistry({
codex: {
default: 'cooldown-default@example.com',
accounts: {
'cooldown-default@example.com': {
email: 'cooldown-default@example.com',
tokenFile: 'codex-cooldown-default.json',
},
'cooldown-exhausted@example.com': {
email: 'cooldown-exhausted@example.com',
tokenFile: 'codex-cooldown-exhausted.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth(
'codex-cooldown-default.json',
'cooldown-default@example.com',
'cooldown-default-token'
);
writeCodexAuth(
'codex-cooldown-exhausted.json',
'cooldown-exhausted@example.com',
'cooldown-exhausted-token'
);
const { applyCooldown, preflightCheck } = await import('../../quota/quota-manager');
applyCooldown('codex', 'cooldown-exhausted@example.com', 10);
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
const usedPercent = accountHeader === 'chatgpt-cooldown-default@example.com' ? 10 : 100;
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const result = await preflightCheck('codex');
const { getAccount } = await import('../account-manager');
expect(result.accountId).toBe('cooldown-default@example.com');
expect(getAccount('codex', 'cooldown-exhausted@example.com')?.paused).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-cooldown-exhausted.json')
)
).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-cooldown-exhausted.json')
)
).toBe(false);
});
it('should reconcile exhausted rotation accounts before honoring forced default', async () => {
writeRegistry({
codex: {
default: 'forced-default@example.com',
accounts: {
'forced-default@example.com': {
email: 'forced-default@example.com',
tokenFile: 'codex-forced-default.json',
},
'forced-exhausted@example.com': {
email: 'forced-exhausted@example.com',
tokenFile: 'codex-forced-exhausted.json',
},
'forced-healthy@example.com': {
email: 'forced-healthy@example.com',
tokenFile: 'codex-forced-healthy.json',
},
},
},
});
writeConfig({
mode: 'auto',
auto: {
tier_priority: ['ultra', 'pro', 'free'],
exhaustion_threshold: 5,
cooldown_minutes: 10,
preflight_check: true,
},
manual: {
forced_default: 'forced-default@example.com',
},
runtime_monitor: {
enabled: true,
normal_interval_seconds: 300,
critical_interval_seconds: 60,
warn_threshold: 20,
exhaustion_threshold: 5,
cooldown_minutes: 10,
},
});
writeCodexAuth('codex-forced-default.json', 'forced-default@example.com', 'forced-token');
writeCodexAuth(
'codex-forced-exhausted.json',
'forced-exhausted@example.com',
'forced-exhausted-token'
);
writeCodexAuth('codex-forced-healthy.json', 'forced-healthy@example.com', 'healthy-token');
global.fetch = mock((_url: string, options?: RequestInit) => {
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
const usedPercent = accountHeader === 'chatgpt-forced-exhausted@example.com' ? 100 : 10;
return Promise.resolve(
Response.json({
plan_type: 'pro',
rate_limit: {
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
},
})
);
}) as typeof fetch;
const { preflightCheck } = await import('../../quota/quota-manager');
const result = await preflightCheck('codex');
const { getAccount, getDefaultAccount } = await import('../account-manager');
expect(result.accountId).toBe('forced-default@example.com');
expect(result.reason).toBe('Forced default override');
expect(getDefaultAccount('codex')?.id).toBe('forced-default@example.com');
expect(getAccount('codex', 'forced-default@example.com')?.paused).not.toBe(true);
expect(getAccount('codex', 'forced-healthy@example.com')?.paused).not.toBe(true);
expect(getAccount('codex', 'forced-exhausted@example.com')?.paused).toBe(true);
expect(
fs.existsSync(
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-forced-exhausted.json')
)
).toBe(true);
});
it('should not durably self-pause during preflight when fallback quota is unknown', async () => {
writeRegistry({
codex: {
src/cliproxy/quota/quota-manager.ts
+72 -1
View File
@@ -476,6 +476,75 @@ export async function findHealthyAccount(
return selectable[0];
}
export async function reconcileExhaustedRotationAccounts(
provider: CLIProxyProvider
): Promise<string[]> {
if (!isManagedQuotaProvider(provider)) {
return [];
}
const { pauseAccountForQuotaCooldown, restoreExpiredQuotaPauses } = await import(
'../accounts/account-safety'
);
restoreExpiredQuotaPauses();
const config = loadOrCreateUnifiedConfig();
const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5;
const cooldownMinutes = config.quota_management?.auto?.cooldown_minutes ?? 5;
const activeAccounts = getProviderAccounts(provider).filter(
(account) => !isAccountPaused(provider, account.id)
);
if (activeAccounts.length < 2) {
return [];
}
const accountsWithQuota = await batchedMap(
activeAccounts,
async (account) => {
let quota = getCachedQuota(provider, account.id);
if (!quota) {
quota = await fetchQuotaWithDedup(provider, account.id);
}
return {
account,
quotaPercent: calculateQuotaPercent(provider, quota),
};
},
10
);
const healthyAccountIds = new Set(
accountsWithQuota
.filter(({ account, quotaPercent }) => {
if (isOnCooldown(provider, account.id)) return false;
return quotaPercent !== null && quotaPercent >= threshold;
})
.map(({ account }) => account.id)
);
if (healthyAccountIds.size === 0) {
return [];
}
const pausedAccountIds: string[] = [];
for (const { account, quotaPercent } of accountsWithQuota) {
const exhaustedByQuota = quotaPercent !== null && quotaPercent < threshold;
const exhaustedByCooldown = isOnCooldown(provider, account.id);
if (!exhaustedByQuota && !exhaustedByCooldown) continue;
const hasOtherHealthyAccount = [...healthyAccountIds].some((id) => id !== account.id);
if (!hasOtherHealthyAccount) continue;
applyCooldown(provider, account.id, cooldownMinutes);
if (pauseAccountForQuotaCooldown(provider, account.id, cooldownMinutes)) {
pausedAccountIds.push(account.id);
}
}
return pausedAccountIds;
}
/**
* Find and switch to a healthy account
*/
@@ -543,11 +612,13 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
return { proceed: false, accountId: '', reason: 'No accounts configured' };
}
await reconcileExhaustedRotationAccounts(provider);
// Check forced_default override (manual mode)
const forcedDefault = quotaConfig.manual?.forced_default;
if (forcedDefault) {
const forcedAccount = getProviderAccounts(provider).find((a) => a.id === forcedDefault);
if (forcedAccount) {
if (forcedAccount && !isAccountPaused(provider, forcedAccount.id)) {
return { proceed: true, accountId: forcedAccount.id, reason: 'Forced default override' };
}
}