mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-16 20:17:45 +00:00
Merge pull request #1124 from kaitranntt/kai/feat/1123-self-pause-exhausted-accounts
feat(cliproxy): self-pause exhausted quota accounts
This commit is contained in:
@@ -41,6 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic
|
||||
|
||||
### Recent Fixes
|
||||
|
||||
- **2026-04-28**: **#1123** CLIProxy quota failover now uses the dashboard/manual pause mechanism for all quota-visible OAuth providers with CCS quota fetchers: Antigravity, Claude, Codex, Gemini CLI, and GitHub Copilot. When a healthy fallback exists, CCS moves the exhausted account token out of the live `auth/` folder into `auth-paused/`, marks the account paused for dashboard visibility, persists the cooldown for auto-resume, and still avoids self-pausing the last usable account.
|
||||
- **2026-04-28**: **#1115** CCS now exposes upstream CLIProxy session affinity as a first-class local managed setting. Users can inspect and toggle local `session-affinity` plus TTL from `ccs cliproxy routing affinity`, from the `/cliproxy` dashboard routing card, and through the local dashboard API. The generated local CLIProxy config now persists `routing.session-affinity` and `routing.session-affinity-ttl`, help/copy explains that CLIProxy prefers explicit session or thread identifiers before falling back to prompt-history hashing, and remote session-affinity management stays explicitly unsupported until upstream management APIs expose more than `routing.strategy`.
|
||||
- **2026-04-24**: **#1065** Local CLIProxy Plus is available again as an explicit opt-in backend through the community-maintained `kaitranntt/CLIProxyAPIPlus` fork. CCS keeps `original` as the default backend, no longer downgrades saved `backend: plus` configs to `original`, updates Plus release lookups to the maintained fork, and documents Plus as a targeted path for plus-only providers.
|
||||
- **2026-04-21**: CLIProxy quota failover now quarantines exhausted Claude and Antigravity accounts out of live rotation when a healthy fallback exists. CCS persists those quota-triggered pauses across launches, automatically resumes them after the configured cooldown window, and deliberately avoids auto-pausing the last available account so single-account setups still degrade gracefully instead of hard-locking themselves.
|
||||
|
||||
@@ -261,7 +261,7 @@ async function checkRemoteProxyHealth(config: ResolvedProxyConfig): Promise<bool
|
||||
### Overview
|
||||
|
||||
Hybrid quota management enables automatic detection of exhausted accounts and failover to next available account.
|
||||
When CCS detects exhaustion and a healthy fallback exists, it also temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires.
|
||||
When CCS detects exhaustion and a healthy fallback exists, it temporarily pauses the exhausted account out of CLIProxy rotation and automatically resumes that pause after the configured cooldown expires. This durable self-pause uses the same account registry and token movement path as dashboard/manual pause, so the dashboard shows the account as paused and CLIProxy cannot rediscover its token from the live `auth/` folder.
|
||||
|
||||
```
|
||||
+===========================================================================+
|
||||
@@ -287,9 +287,11 @@ When CCS detects exhaustion and a healthy fallback exists, it also temporarily p
|
||||
| +---> Check isPaused flag --> Skip if paused
|
||||
| |
|
||||
| +---> Fetch quota from provider API
|
||||
| | - Gemini: /models endpoint
|
||||
| | - Codex: /api/v1/account
|
||||
| | - Kiro: /api/usage
|
||||
| | - Antigravity: fetchAvailableModels
|
||||
| | - Claude: policy limits endpoint
|
||||
| | - Codex: ChatGPT usage windows
|
||||
| | - Gemini CLI: Code Assist quota buckets
|
||||
| | - GitHub Copilot: copilot_internal/user snapshots
|
||||
| |
|
||||
| +---> Detect tier (free/paid/unknown)
|
||||
| |
|
||||
|
||||
@@ -690,6 +690,9 @@ export async function handleQuotaExhaustion(
|
||||
// Dynamic imports to avoid circular dependencies
|
||||
const { applyCooldown, findHealthyAccount } = await import('./quota-manager');
|
||||
const { setDefaultAccount, touchAccount } = await import('./account-manager');
|
||||
const { loadOrCreateUnifiedConfig } = await import('../config/unified-config-loader');
|
||||
const config = loadOrCreateUnifiedConfig();
|
||||
const threshold = config.quota_management?.auto?.exhaustion_threshold ?? 5;
|
||||
|
||||
// Apply cooldown to exhausted account
|
||||
applyCooldown(provider, accountId, cooldownMinutes);
|
||||
@@ -698,7 +701,9 @@ export async function handleQuotaExhaustion(
|
||||
const alternative = await findHealthyAccount(provider, [accountId]);
|
||||
|
||||
if (alternative) {
|
||||
pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes);
|
||||
if (alternative.lastQuota !== null && alternative.lastQuota >= threshold) {
|
||||
pauseAccountForQuotaCooldown(provider, accountId, cooldownMinutes);
|
||||
}
|
||||
setDefaultAccount(provider, alternative.id);
|
||||
touchAccount(provider, alternative.id);
|
||||
writeQuotaExhausted(accountId, alternative.id, cooldownMinutes);
|
||||
|
||||
@@ -106,6 +106,7 @@ import {
|
||||
handleTokenExpiration,
|
||||
handleQuotaCheck,
|
||||
} from './retry-handler';
|
||||
import { MANAGED_QUOTA_PROVIDERS, type ManagedQuotaProvider } from '../quota-manager';
|
||||
import { checkOrJoinProxy, registerProxySession, setupCleanupHandlers } from './session-bridge';
|
||||
import { parseThinkingOverride } from './thinking-arg-parser';
|
||||
import {
|
||||
@@ -128,6 +129,25 @@ import {
|
||||
} from './thinking-override-resolver';
|
||||
import { shouldStartHttpsTunnel } from './https-tunnel-policy';
|
||||
|
||||
function resolveRuntimeQuotaMonitorProviders(
|
||||
provider: CLIProxyProvider,
|
||||
compositeProviders: CLIProxyProvider[]
|
||||
): ManagedQuotaProvider[] {
|
||||
const candidates = compositeProviders.length > 0 ? compositeProviders : [provider];
|
||||
const resolved: ManagedQuotaProvider[] = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
MANAGED_QUOTA_PROVIDERS.includes(candidate as ManagedQuotaProvider) &&
|
||||
!resolved.includes(candidate as ManagedQuotaProvider)
|
||||
) {
|
||||
resolved.push(candidate as ManagedQuotaProvider);
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Default executor configuration */
|
||||
const DEFAULT_CONFIG: ExecutorConfig = {
|
||||
port: CLIPROXY_DEFAULT_PORT,
|
||||
@@ -888,8 +908,7 @@ export async function execClaudeWithCLIProxy(
|
||||
if (!skipLocalAuth) {
|
||||
// Multi-tier quota check for composite variants (check if any tier uses a managed provider)
|
||||
if (compositeProviders.length > 0) {
|
||||
const managedQuotaProviders = ['agy', 'claude'] as const;
|
||||
for (const managedProvider of managedQuotaProviders) {
|
||||
for (const managedProvider of MANAGED_QUOTA_PROVIDERS) {
|
||||
if (compositeProviders.includes(managedProvider)) {
|
||||
await handleQuotaCheck(managedProvider);
|
||||
}
|
||||
@@ -1376,9 +1395,14 @@ export async function execClaudeWithCLIProxy(
|
||||
// 12b. Start runtime quota monitor (adaptive polling during session)
|
||||
if (!skipLocalAuth) {
|
||||
const { startQuotaMonitor } = await import('../quota-manager');
|
||||
const monitorAccount = getDefaultAccount(provider);
|
||||
if (monitorAccount) {
|
||||
startQuotaMonitor(provider, monitorAccount.id);
|
||||
for (const monitorProvider of resolveRuntimeQuotaMonitorProviders(
|
||||
provider,
|
||||
compositeProviders
|
||||
)) {
|
||||
const monitorAccount = getDefaultAccount(monitorProvider);
|
||||
if (monitorAccount) {
|
||||
startQuotaMonitor(monitorProvider, monitorAccount.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1397,4 +1421,8 @@ export async function execClaudeWithCLIProxy(
|
||||
// Re-export utility functions for backwards compatibility
|
||||
export { isPortAvailable, findAvailablePort } from './lifecycle-manager';
|
||||
|
||||
export const __testExports = {
|
||||
resolveRuntimeQuotaMonitorProviders,
|
||||
};
|
||||
|
||||
export default execClaudeWithCLIProxy;
|
||||
|
||||
@@ -80,9 +80,9 @@ export async function handleTokenExpiration(
|
||||
* Handle quota check and auto-switching for providers with quota-based rotation.
|
||||
*/
|
||||
export async function handleQuotaCheck(provider: CLIProxyProvider): Promise<void> {
|
||||
if (provider !== 'agy' && provider !== 'claude') return;
|
||||
const { isManagedQuotaProvider, preflightCheck } = await import('../quota-manager');
|
||||
if (!isManagedQuotaProvider(provider)) return;
|
||||
|
||||
const { preflightCheck } = await import('../quota-manager');
|
||||
const preflight = await preflightCheck(provider);
|
||||
|
||||
if (!preflight.proceed) {
|
||||
|
||||
@@ -48,29 +48,41 @@ interface TokenData {
|
||||
}
|
||||
|
||||
function normalizeSnapshot(raw?: RawGhcpQuotaSnapshot): GhcpQuotaSnapshot {
|
||||
const unlimited = Boolean(raw?.unlimited);
|
||||
const hasPercentSignal =
|
||||
typeof raw?.percent_remaining === 'number' && Number.isFinite(raw.percent_remaining);
|
||||
const rawRemaining = typeof raw?.remaining === 'number' ? raw.remaining : raw?.quota_remaining;
|
||||
const hasEntitlementRemainingSignal =
|
||||
typeof raw?.entitlement === 'number' &&
|
||||
Number.isFinite(raw.entitlement) &&
|
||||
raw.entitlement > 0 &&
|
||||
typeof rawRemaining === 'number' &&
|
||||
Number.isFinite(rawRemaining);
|
||||
const reported =
|
||||
raw !== undefined && (unlimited || hasPercentSignal || hasEntitlementRemainingSignal);
|
||||
const entitlement = Number(raw?.entitlement ?? 0);
|
||||
const remainingRaw = raw?.remaining ?? raw?.quota_remaining ?? 0;
|
||||
const remainingRaw = rawRemaining ?? 0;
|
||||
const remaining = Number(remainingRaw);
|
||||
const safeEntitlement = Number.isFinite(entitlement) ? Math.max(0, entitlement) : 0;
|
||||
const safeRemaining = Number.isFinite(remaining) ? Math.max(0, remaining) : 0;
|
||||
const used = Math.max(0, safeEntitlement - safeRemaining);
|
||||
|
||||
const percentRemainingRaw =
|
||||
typeof raw?.percent_remaining === 'number' ? raw.percent_remaining : null;
|
||||
const percentRemaining =
|
||||
percentRemainingRaw !== null
|
||||
? clampPercent(percentRemainingRaw)
|
||||
: safeEntitlement > 0
|
||||
const percentRemaining = unlimited
|
||||
? 100
|
||||
: hasPercentSignal
|
||||
? clampPercent(raw.percent_remaining as number)
|
||||
: reported && safeEntitlement > 0
|
||||
? clampPercent((safeRemaining / safeEntitlement) * 100)
|
||||
: 0;
|
||||
|
||||
return {
|
||||
reported,
|
||||
entitlement: safeEntitlement,
|
||||
remaining: safeRemaining,
|
||||
used,
|
||||
percentRemaining,
|
||||
percentUsed: clampPercent(100 - percentRemaining),
|
||||
unlimited: Boolean(raw?.unlimited),
|
||||
unlimited,
|
||||
overageCount:
|
||||
typeof raw?.overage_count === 'number' && Number.isFinite(raw.overage_count)
|
||||
? Math.max(0, raw.overage_count)
|
||||
|
||||
+219
-67
@@ -15,7 +15,15 @@
|
||||
import { CLIProxyProvider } from './types';
|
||||
import { QuotaResult, fetchAccountQuota } from './quota-fetcher';
|
||||
import { fetchClaudeQuota } from './quota-fetcher-claude';
|
||||
import type { ClaudeQuotaResult } from './quota-types';
|
||||
import { fetchCodexQuota } from './quota-fetcher-codex';
|
||||
import { fetchGeminiCliQuota } from './quota-fetcher-gemini-cli';
|
||||
import { fetchGhcpQuota } from './quota-fetcher-ghcp';
|
||||
import type {
|
||||
ClaudeQuotaResult,
|
||||
CodexQuotaResult,
|
||||
GeminiCliQuotaResult,
|
||||
GhcpQuotaResult,
|
||||
} from './quota-types';
|
||||
import {
|
||||
getDefaultAccount,
|
||||
getProviderAccounts,
|
||||
@@ -27,12 +35,25 @@ import {
|
||||
import { loadOrCreateUnifiedConfig } from '../config/unified-config-loader';
|
||||
import type { RuntimeMonitorConfig } from '../config/unified-config-types';
|
||||
|
||||
type ManagedQuotaProvider = 'agy' | 'claude';
|
||||
type ManagedQuotaResult = QuotaResult | ClaudeQuotaResult;
|
||||
export type ManagedQuotaProvider = 'agy' | 'claude' | 'codex' | 'gemini' | 'ghcp';
|
||||
type ManagedQuotaResult =
|
||||
| QuotaResult
|
||||
| ClaudeQuotaResult
|
||||
| CodexQuotaResult
|
||||
| GeminiCliQuotaResult
|
||||
| GhcpQuotaResult;
|
||||
|
||||
const MANAGED_QUOTA_PROVIDERS: readonly ManagedQuotaProvider[] = ['agy', 'claude'];
|
||||
export const MANAGED_QUOTA_PROVIDERS: readonly ManagedQuotaProvider[] = [
|
||||
'agy',
|
||||
'claude',
|
||||
'codex',
|
||||
'gemini',
|
||||
'ghcp',
|
||||
];
|
||||
|
||||
function isManagedQuotaProvider(provider: CLIProxyProvider): provider is ManagedQuotaProvider {
|
||||
export function isManagedQuotaProvider(
|
||||
provider: CLIProxyProvider
|
||||
): provider is ManagedQuotaProvider {
|
||||
return MANAGED_QUOTA_PROVIDERS.includes(provider as ManagedQuotaProvider);
|
||||
}
|
||||
|
||||
@@ -117,19 +138,7 @@ async function fetchQuotaWithDedup(
|
||||
setCachedQuota(provider, accountId, result);
|
||||
return result;
|
||||
})
|
||||
.catch((): ManagedQuotaResult => {
|
||||
if (provider === 'claude') {
|
||||
return {
|
||||
success: false,
|
||||
windows: [],
|
||||
coreUsage: { fiveHour: null, weekly: null },
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Failed to fetch Claude quota',
|
||||
accountId,
|
||||
};
|
||||
}
|
||||
return { success: false, models: [], lastUpdated: Date.now() };
|
||||
})
|
||||
.catch((): ManagedQuotaResult => buildFailedQuotaResult(provider, accountId))
|
||||
.finally(() => {
|
||||
pendingFetches.delete(key);
|
||||
});
|
||||
@@ -143,10 +152,85 @@ async function fetchManagedQuota(
|
||||
accountId: string,
|
||||
verbose: boolean
|
||||
): Promise<ManagedQuotaResult> {
|
||||
if (provider === 'claude') {
|
||||
return fetchClaudeQuota(accountId, verbose);
|
||||
switch (provider) {
|
||||
case 'agy':
|
||||
return fetchAccountQuota(provider, accountId, verbose);
|
||||
case 'claude':
|
||||
return fetchClaudeQuota(accountId, verbose);
|
||||
case 'codex':
|
||||
return fetchCodexQuota(accountId, verbose);
|
||||
case 'gemini':
|
||||
return fetchGeminiCliQuota(accountId, verbose);
|
||||
case 'ghcp':
|
||||
return fetchGhcpQuota(accountId, verbose);
|
||||
}
|
||||
}
|
||||
|
||||
function buildFailedGhcpSnapshot(): GhcpQuotaResult['snapshots']['chat'] {
|
||||
return {
|
||||
reported: false,
|
||||
entitlement: 0,
|
||||
remaining: 0,
|
||||
used: 0,
|
||||
percentRemaining: 0,
|
||||
percentUsed: 0,
|
||||
unlimited: false,
|
||||
overageCount: 0,
|
||||
overagePermitted: false,
|
||||
quotaId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function buildFailedQuotaResult(
|
||||
provider: ManagedQuotaProvider,
|
||||
accountId: string
|
||||
): ManagedQuotaResult {
|
||||
switch (provider) {
|
||||
case 'agy':
|
||||
return { success: false, models: [], lastUpdated: Date.now(), accountId };
|
||||
case 'claude':
|
||||
return {
|
||||
success: false,
|
||||
windows: [],
|
||||
coreUsage: { fiveHour: null, weekly: null },
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Failed to fetch Claude quota',
|
||||
accountId,
|
||||
};
|
||||
case 'codex':
|
||||
return {
|
||||
success: false,
|
||||
windows: [],
|
||||
coreUsage: { fiveHour: null, weekly: null },
|
||||
planType: null,
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Failed to fetch Codex quota',
|
||||
accountId,
|
||||
};
|
||||
case 'gemini':
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: null,
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Failed to fetch Gemini quota',
|
||||
accountId,
|
||||
};
|
||||
case 'ghcp':
|
||||
return {
|
||||
success: false,
|
||||
planType: null,
|
||||
quotaResetDate: null,
|
||||
snapshots: {
|
||||
premiumInteractions: buildFailedGhcpSnapshot(),
|
||||
chat: buildFailedGhcpSnapshot(),
|
||||
completions: buildFailedGhcpSnapshot(),
|
||||
},
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Failed to fetch GitHub Copilot quota',
|
||||
accountId,
|
||||
};
|
||||
}
|
||||
return fetchAccountQuota(provider, accountId, verbose);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -264,14 +348,62 @@ function calculateClaudeQuotaPercent(quota: ClaudeQuotaResult): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function calculateCodexQuotaPercent(quota: CodexQuotaResult): number | null {
|
||||
if (!quota.success) return null;
|
||||
|
||||
const coreWindows = [quota.coreUsage?.fiveHour, quota.coreUsage?.weekly].filter(
|
||||
(window): window is NonNullable<typeof window> => !!window
|
||||
);
|
||||
if (coreWindows.length > 0) {
|
||||
return Math.min(...coreWindows.map((window) => window.remainingPercent));
|
||||
}
|
||||
|
||||
const usageWindows = quota.windows.filter((window) => window.category !== 'code-review');
|
||||
if (usageWindows.length > 0) {
|
||||
return Math.min(...usageWindows.map((window) => window.remainingPercent));
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function calculateGeminiQuotaPercent(quota: GeminiCliQuotaResult): number | null {
|
||||
if (!quota.success || quota.buckets.length === 0) return null;
|
||||
return Math.min(...quota.buckets.map((bucket) => bucket.remainingPercent));
|
||||
}
|
||||
|
||||
function calculateGhcpQuotaPercent(quota: GhcpQuotaResult): number | null {
|
||||
if (!quota.success) return null;
|
||||
const percentages = [
|
||||
quota.snapshots.premiumInteractions,
|
||||
quota.snapshots.chat,
|
||||
quota.snapshots.completions,
|
||||
]
|
||||
.filter((snapshot) => snapshot.reported !== false)
|
||||
.map((snapshot) => (snapshot.unlimited ? 100 : snapshot.percentRemaining))
|
||||
.filter((percentage) => Number.isFinite(percentage));
|
||||
|
||||
return percentages.length > 0 ? Math.min(...percentages) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate normalized quota percentage for managed providers.
|
||||
*/
|
||||
function calculateQuotaPercent(quota: ManagedQuotaResult): number | null {
|
||||
if ('models' in quota) {
|
||||
return calculateAgyQuotaPercent(quota);
|
||||
function calculateQuotaPercent(
|
||||
provider: ManagedQuotaProvider,
|
||||
quota: ManagedQuotaResult
|
||||
): number | null {
|
||||
switch (provider) {
|
||||
case 'agy':
|
||||
return calculateAgyQuotaPercent(quota as QuotaResult);
|
||||
case 'claude':
|
||||
return calculateClaudeQuotaPercent(quota as ClaudeQuotaResult);
|
||||
case 'codex':
|
||||
return calculateCodexQuotaPercent(quota as CodexQuotaResult);
|
||||
case 'gemini':
|
||||
return calculateGeminiQuotaPercent(quota as GeminiCliQuotaResult);
|
||||
case 'ghcp':
|
||||
return calculateGhcpQuotaPercent(quota as GhcpQuotaResult);
|
||||
}
|
||||
return calculateClaudeQuotaPercent(quota);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -312,7 +444,7 @@ export async function findHealthyAccount(
|
||||
quota = await fetchQuotaWithDedup(provider, account.id);
|
||||
}
|
||||
|
||||
const avgQuota = calculateQuotaPercent(quota);
|
||||
const avgQuota = calculateQuotaPercent(provider, quota);
|
||||
|
||||
return {
|
||||
id: account.id,
|
||||
@@ -437,7 +569,7 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
|
||||
|
||||
// Calculate normalized quota percentage.
|
||||
// Null means quota data is unavailable (e.g. provider does not expose limits for this account).
|
||||
const quotaPercent = calculateQuotaPercent(quota);
|
||||
const quotaPercent = calculateQuotaPercent(provider, quota);
|
||||
if (quotaPercent === null) {
|
||||
return {
|
||||
proceed: true,
|
||||
@@ -462,11 +594,13 @@ export async function preflightCheck(provider: CLIProxyProvider): Promise<Prefli
|
||||
};
|
||||
}
|
||||
|
||||
pauseAccountForQuotaCooldown(
|
||||
provider,
|
||||
defaultAccount.id,
|
||||
quotaConfig.auto?.cooldown_minutes ?? 5
|
||||
);
|
||||
if (alternative.lastQuota !== null && alternative.lastQuota >= threshold) {
|
||||
pauseAccountForQuotaCooldown(
|
||||
provider,
|
||||
defaultAccount.id,
|
||||
quotaConfig.auto?.cooldown_minutes ?? 5
|
||||
);
|
||||
}
|
||||
setDefaultAccount(provider, alternative.id);
|
||||
touchAccount(provider, alternative.id);
|
||||
return {
|
||||
@@ -506,12 +640,14 @@ export async function getQuotaStatus(provider: CLIProxyProvider): Promise<{
|
||||
|
||||
const results = await Promise.all(
|
||||
accounts.map(async (account) => {
|
||||
const managedProvider = isManagedQuotaProvider(provider) ? provider : null;
|
||||
let quota = getCachedQuota(provider, account.id);
|
||||
if (!quota && isManagedQuotaProvider(provider)) {
|
||||
quota = await fetchQuotaWithDedup(provider, account.id);
|
||||
if (!quota && managedProvider) {
|
||||
quota = await fetchQuotaWithDedup(managedProvider, account.id);
|
||||
}
|
||||
|
||||
const avgQuota = quota ? calculateQuotaPercent(quota) : null;
|
||||
const avgQuota =
|
||||
quota && managedProvider ? calculateQuotaPercent(managedProvider, quota) : null;
|
||||
|
||||
return {
|
||||
account,
|
||||
@@ -530,14 +666,14 @@ export async function getQuotaStatus(provider: CLIProxyProvider): Promise<{
|
||||
// RUNTIME QUOTA MONITOR (adaptive polling during active sessions)
|
||||
// ============================================================================
|
||||
|
||||
/** Active monitor timer (null = not running) */
|
||||
let monitorTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
interface RuntimeMonitorState {
|
||||
timer: ReturnType<typeof setTimeout> | null;
|
||||
hasWarnedThisSession: boolean;
|
||||
stopped: boolean;
|
||||
}
|
||||
|
||||
/** Tracks if warning was shown this session (avoid spam) */
|
||||
let hasWarnedThisSession = false;
|
||||
|
||||
/** Guards against in-flight poll callbacks running after stop */
|
||||
let monitorStopped = false;
|
||||
/** Active monitor timers keyed by provider/account. */
|
||||
const activeRuntimeMonitors = new Map<string, RuntimeMonitorState>();
|
||||
|
||||
/**
|
||||
* Schedule next quota poll with adaptive interval.
|
||||
@@ -547,16 +683,18 @@ function scheduleNextPoll(
|
||||
provider: ManagedQuotaProvider,
|
||||
accountId: string,
|
||||
monitorConfig: RuntimeMonitorConfig,
|
||||
intervalMs: number
|
||||
intervalMs: number,
|
||||
state: RuntimeMonitorState
|
||||
): void {
|
||||
monitorTimer = setTimeout(async () => {
|
||||
const monitorKey = getCacheKey(provider, accountId);
|
||||
state.timer = setTimeout(async () => {
|
||||
// Guard: skip if monitor was stopped while this callback was queued
|
||||
if (monitorStopped) return;
|
||||
if (state.stopped || activeRuntimeMonitors.get(monitorKey) !== state) return;
|
||||
|
||||
try {
|
||||
const quota = await fetchQuotaWithDedup(provider, accountId);
|
||||
if (monitorStopped) return; // Re-check after async fetch
|
||||
const avgQuota = calculateQuotaPercent(quota);
|
||||
if (state.stopped || activeRuntimeMonitors.get(monitorKey) !== state) return;
|
||||
const avgQuota = calculateQuotaPercent(provider, quota);
|
||||
|
||||
if (avgQuota === null) {
|
||||
// Quota data unavailable: keep polling, but do not treat unknown as healthy/exhausted.
|
||||
@@ -564,7 +702,8 @@ function scheduleNextPoll(
|
||||
provider,
|
||||
accountId,
|
||||
monitorConfig,
|
||||
monitorConfig.normal_interval_seconds * 1000
|
||||
monitorConfig.normal_interval_seconds * 1000,
|
||||
state
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -576,22 +715,25 @@ function scheduleNextPoll(
|
||||
// default only takes effect on next session start via preflightCheck().
|
||||
const { handleQuotaExhaustion } = await import('./account-safety');
|
||||
await handleQuotaExhaustion(provider, accountId, monitorConfig.cooldown_minutes);
|
||||
monitorTimer = null;
|
||||
state.stopped = true;
|
||||
state.timer = null;
|
||||
activeRuntimeMonitors.delete(monitorKey);
|
||||
return; // Stop polling
|
||||
}
|
||||
|
||||
if (avgQuota <= monitorConfig.warn_threshold) {
|
||||
// WARNING: switch to critical interval, warn once
|
||||
if (!hasWarnedThisSession) {
|
||||
if (!state.hasWarnedThisSession) {
|
||||
const { writeQuotaWarning } = await import('./account-safety');
|
||||
writeQuotaWarning(accountId, avgQuota);
|
||||
hasWarnedThisSession = true;
|
||||
state.hasWarnedThisSession = true;
|
||||
}
|
||||
scheduleNextPoll(
|
||||
provider,
|
||||
accountId,
|
||||
monitorConfig,
|
||||
monitorConfig.critical_interval_seconds * 1000
|
||||
monitorConfig.critical_interval_seconds * 1000,
|
||||
state
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -601,17 +743,20 @@ function scheduleNextPoll(
|
||||
provider,
|
||||
accountId,
|
||||
monitorConfig,
|
||||
monitorConfig.normal_interval_seconds * 1000
|
||||
monitorConfig.normal_interval_seconds * 1000,
|
||||
state
|
||||
);
|
||||
} catch {
|
||||
// API failure: silently reschedule at same interval
|
||||
scheduleNextPoll(provider, accountId, monitorConfig, intervalMs);
|
||||
if (!state.stopped && activeRuntimeMonitors.get(monitorKey) === state) {
|
||||
scheduleNextPoll(provider, accountId, monitorConfig, intervalMs, state);
|
||||
}
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
// Prevent monitor from keeping Node.js process alive
|
||||
if (monitorTimer && typeof monitorTimer === 'object' && 'unref' in monitorTimer) {
|
||||
monitorTimer.unref();
|
||||
if (state.timer && typeof state.timer === 'object' && 'unref' in state.timer) {
|
||||
state.timer.unref();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -628,9 +773,6 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
|
||||
// Only managed providers support runtime quota monitoring.
|
||||
if (!isManagedQuotaProvider(provider)) return;
|
||||
|
||||
// Prevent duplicate monitors
|
||||
if (monitorTimer) return;
|
||||
|
||||
const config = loadOrCreateUnifiedConfig();
|
||||
const quotaConfig = config.quota_management;
|
||||
|
||||
@@ -647,15 +789,23 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
|
||||
return; // Invalid config — skip monitoring silently (logged at config level)
|
||||
}
|
||||
|
||||
hasWarnedThisSession = false;
|
||||
monitorStopped = false;
|
||||
const monitorKey = getCacheKey(provider, accountId);
|
||||
if (activeRuntimeMonitors.has(monitorKey)) return;
|
||||
|
||||
const state: RuntimeMonitorState = {
|
||||
timer: null,
|
||||
hasWarnedThisSession: false,
|
||||
stopped: false,
|
||||
};
|
||||
activeRuntimeMonitors.set(monitorKey, state);
|
||||
|
||||
// Start first poll at normal interval
|
||||
scheduleNextPoll(
|
||||
provider,
|
||||
accountId,
|
||||
quotaConfig.runtime_monitor,
|
||||
quotaConfig.runtime_monitor.normal_interval_seconds * 1000
|
||||
quotaConfig.runtime_monitor.normal_interval_seconds * 1000,
|
||||
state
|
||||
);
|
||||
}
|
||||
|
||||
@@ -663,10 +813,12 @@ export function startQuotaMonitor(provider: CLIProxyProvider, accountId: string)
|
||||
* Stop the runtime quota monitor. Safe to call multiple times.
|
||||
*/
|
||||
export function stopQuotaMonitor(): void {
|
||||
monitorStopped = true;
|
||||
if (monitorTimer) {
|
||||
clearTimeout(monitorTimer);
|
||||
monitorTimer = null;
|
||||
for (const state of activeRuntimeMonitors.values()) {
|
||||
state.stopped = true;
|
||||
if (state.timer) {
|
||||
clearTimeout(state.timer);
|
||||
state.timer = null;
|
||||
}
|
||||
}
|
||||
hasWarnedThisSession = false;
|
||||
activeRuntimeMonitors.clear();
|
||||
}
|
||||
|
||||
@@ -230,6 +230,8 @@ export interface GeminiCliQuotaResult extends QuotaErrorMetadata {
|
||||
* GitHub Copilot quota snapshot.
|
||||
*/
|
||||
export interface GhcpQuotaSnapshot {
|
||||
/** False when upstream omitted this quota category; callers should exclude it from health math */
|
||||
reported?: boolean;
|
||||
/** Total quota allocation for this category */
|
||||
entitlement: number;
|
||||
/** Remaining quota count */
|
||||
|
||||
@@ -752,13 +752,17 @@ function displayGhcpQuotaSection(results: { account: string; quota: GhcpQuotaRes
|
||||
continue;
|
||||
}
|
||||
|
||||
const rows = [
|
||||
quota.snapshots.premiumInteractions.percentRemaining,
|
||||
quota.snapshots.chat.percentRemaining,
|
||||
quota.snapshots.completions.percentRemaining,
|
||||
];
|
||||
const minQuota = rows.length > 0 ? Math.min(...rows) : 0;
|
||||
const statusIcon = minQuota > 50 ? ok('') : minQuota > 10 ? warn('') : fail('');
|
||||
const reportedSnapshots = [
|
||||
quota.snapshots.premiumInteractions,
|
||||
quota.snapshots.chat,
|
||||
quota.snapshots.completions,
|
||||
].filter((snapshot) => snapshot.reported !== false);
|
||||
const rows = reportedSnapshots.map((snapshot) =>
|
||||
snapshot.unlimited ? 100 : snapshot.percentRemaining
|
||||
);
|
||||
const minQuota = rows.length > 0 ? Math.min(...rows) : null;
|
||||
const statusIcon =
|
||||
minQuota === null ? info('') : minQuota > 50 ? ok('') : minQuota > 10 ? warn('') : fail('');
|
||||
const planBadge = quota.planType ? color(` [${quota.planType}]`, 'info') : '';
|
||||
|
||||
console.log(` ${statusIcon}${accountLabel}${defaultMark}${planBadge}`);
|
||||
@@ -766,12 +770,14 @@ function displayGhcpQuotaSection(results: { account: string; quota: GhcpQuotaRes
|
||||
console.log(` ${dim(`Resets ${formatResetTimeISO(quota.quotaResetDate)}`)}`);
|
||||
}
|
||||
|
||||
const items: Array<[string, GhcpQuotaResult['snapshots'][keyof GhcpQuotaResult['snapshots']]]> =
|
||||
[
|
||||
['Premium interactions', quota.snapshots.premiumInteractions],
|
||||
['Chat', quota.snapshots.chat],
|
||||
['Completions', quota.snapshots.completions],
|
||||
];
|
||||
const allItems: Array<
|
||||
[string, GhcpQuotaResult['snapshots'][keyof GhcpQuotaResult['snapshots']]]
|
||||
> = [
|
||||
['Premium interactions', quota.snapshots.premiumInteractions],
|
||||
['Chat', quota.snapshots.chat],
|
||||
['Completions', quota.snapshots.completions],
|
||||
];
|
||||
const items = allItems.filter(([, snapshot]) => snapshot.reported !== false);
|
||||
|
||||
for (const [label, snapshot] of items) {
|
||||
const bar = formatQuotaBar(snapshot.percentRemaining);
|
||||
|
||||
@@ -86,6 +86,36 @@ function writeClaudeAuth(accountId: string, accessToken: string): void {
|
||||
);
|
||||
}
|
||||
|
||||
function writeCodexAuth(tokenFile: string, accountId: string, accessToken: string): void {
|
||||
writeAuthToken(tokenFile, {
|
||||
access_token: accessToken,
|
||||
account_id: `chatgpt-${accountId}`,
|
||||
expired: '2099-01-01T00:00:00.000Z',
|
||||
type: 'codex',
|
||||
email: accountId,
|
||||
});
|
||||
}
|
||||
|
||||
function writeGhcpAuth(tokenFile: string, accessToken: string): void {
|
||||
writeAuthToken(tokenFile, {
|
||||
access_token: accessToken,
|
||||
type: 'ghcp',
|
||||
});
|
||||
}
|
||||
|
||||
function writeGeminiAuth(tokenFile: string, accountId: string, accessToken: string): void {
|
||||
writeAuthToken(tokenFile, {
|
||||
type: 'gemini',
|
||||
email: accountId,
|
||||
project_id: 'cloudaicompanion-test-123',
|
||||
token: {
|
||||
access_token: accessToken,
|
||||
refresh_token: `${accessToken}-refresh`,
|
||||
expiry: Date.now() + 60 * 60 * 1000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function writeAuthToken(tokenFile: string, payload: Record<string, unknown>): void {
|
||||
const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth');
|
||||
fs.mkdirSync(authDir, { recursive: true });
|
||||
@@ -264,7 +294,7 @@ describe('Quota Exhaustion Handlers', () => {
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('should switch Claude accounts when fallback quota endpoint returns 404', async () => {
|
||||
it('should switch Claude accounts without durable pause when fallback quota is unknown', async () => {
|
||||
writeRegistry({
|
||||
claude: {
|
||||
default: 'exhausted@example.com',
|
||||
@@ -312,11 +342,27 @@ describe('Quota Exhaustion Handlers', () => {
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await handleQuotaExhaustion('claude', 'exhausted@example.com', 10);
|
||||
const { getDefaultAccount } = await import('../../../src/cliproxy/account-manager');
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedTo).toBe('fallback@example.com');
|
||||
expect(getDefaultAccount('claude')?.id).toBe('fallback@example.com');
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true);
|
||||
expect(getAccount('claude', 'exhausted@example.com')?.paused).not.toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(
|
||||
false
|
||||
);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
tmpDir,
|
||||
'.ccs',
|
||||
'cliproxy',
|
||||
'auth',
|
||||
`claude-${sanitizeEmail('exhausted@example.com')}.json`
|
||||
)
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
@@ -327,6 +373,512 @@ describe('Quota Exhaustion Handlers', () => {
|
||||
`claude-${sanitizeEmail('exhausted@example.com')}.json`
|
||||
)
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('should self-pause exhausted Codex accounts when a healthy fallback exists', async () => {
|
||||
writeRegistry({
|
||||
codex: {
|
||||
default: 'exhausted@example.com',
|
||||
accounts: {
|
||||
'exhausted@example.com': {
|
||||
email: 'exhausted@example.com',
|
||||
tokenFile: 'codex-exhausted.json',
|
||||
},
|
||||
'fallback@example.com': {
|
||||
email: 'fallback@example.com',
|
||||
tokenFile: 'codex-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeCodexAuth('codex-exhausted.json', 'exhausted@example.com', 'exhausted-token');
|
||||
writeCodexAuth('codex-fallback.json', 'fallback@example.com', 'fallback-token');
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
|
||||
const usedPercent = accountHeader === 'chatgpt-fallback@example.com' ? 10 : 100;
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
plan_type: 'pro',
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
|
||||
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
|
||||
},
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await handleQuotaExhaustion('codex', 'exhausted@example.com', 10);
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedTo).toBe('fallback@example.com');
|
||||
expect(getDefaultAccount('codex')?.id).toBe('fallback@example.com');
|
||||
expect(getAccount('codex', 'exhausted@example.com')?.paused).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-exhausted.json'))
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-exhausted.json'))
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('should not durably self-pause the only exhausted Codex account during preflight', async () => {
|
||||
writeRegistry({
|
||||
codex: {
|
||||
default: 'only-codex@example.com',
|
||||
accounts: {
|
||||
'only-codex@example.com': {
|
||||
email: 'only-codex@example.com',
|
||||
tokenFile: 'codex-only.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeCodexAuth('codex-only.json', 'only-codex@example.com', 'only-token');
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve(
|
||||
Response.json({
|
||||
plan_type: 'pro',
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: 100, reset_after_seconds: 3600 },
|
||||
secondary_window: { used_percent: 100, reset_after_seconds: 604800 },
|
||||
},
|
||||
})
|
||||
)
|
||||
) as typeof fetch;
|
||||
|
||||
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
|
||||
const result = await preflightCheck('codex');
|
||||
const { getAccount } = await import('../../../src/cliproxy/account-manager');
|
||||
|
||||
expect(result.accountId).toBe('only-codex@example.com');
|
||||
expect(result.switchedFrom).toBeUndefined();
|
||||
expect(result.reason).toContain('no alternatives available');
|
||||
expect(getAccount('codex', 'only-codex@example.com')?.paused).not.toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-only.json'))).toBe(
|
||||
true
|
||||
);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-only.json'))
|
||||
).toBe(false);
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('should self-pause an exhausted Codex default during preflight when fallback exists', async () => {
|
||||
writeRegistry({
|
||||
codex: {
|
||||
default: 'preflight-exhausted@example.com',
|
||||
accounts: {
|
||||
'preflight-exhausted@example.com': {
|
||||
email: 'preflight-exhausted@example.com',
|
||||
tokenFile: 'codex-preflight-exhausted.json',
|
||||
},
|
||||
'preflight-fallback@example.com': {
|
||||
email: 'preflight-fallback@example.com',
|
||||
tokenFile: 'codex-preflight-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeCodexAuth(
|
||||
'codex-preflight-exhausted.json',
|
||||
'preflight-exhausted@example.com',
|
||||
'preflight-exhausted-token'
|
||||
);
|
||||
writeCodexAuth(
|
||||
'codex-preflight-fallback.json',
|
||||
'preflight-fallback@example.com',
|
||||
'preflight-fallback-token'
|
||||
);
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
|
||||
const usedPercent = accountHeader === 'chatgpt-preflight-fallback@example.com' ? 10 : 100;
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
plan_type: 'pro',
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: usedPercent, reset_after_seconds: 3600 },
|
||||
secondary_window: { used_percent: usedPercent, reset_after_seconds: 604800 },
|
||||
},
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
|
||||
const result = await preflightCheck('codex');
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedFrom).toBe('preflight-exhausted@example.com');
|
||||
expect(result.accountId).toBe('preflight-fallback@example.com');
|
||||
expect(getDefaultAccount('codex')?.id).toBe('preflight-fallback@example.com');
|
||||
expect(getAccount('codex', 'preflight-exhausted@example.com')?.paused).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'codex-preflight-exhausted.json')
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('should not durably self-pause during preflight when fallback quota is unknown', async () => {
|
||||
writeRegistry({
|
||||
codex: {
|
||||
default: 'preflight-unknown-exhausted@example.com',
|
||||
accounts: {
|
||||
'preflight-unknown-exhausted@example.com': {
|
||||
email: 'preflight-unknown-exhausted@example.com',
|
||||
tokenFile: 'codex-preflight-unknown-exhausted.json',
|
||||
},
|
||||
'preflight-unknown-fallback@example.com': {
|
||||
email: 'preflight-unknown-fallback@example.com',
|
||||
tokenFile: 'codex-preflight-unknown-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeCodexAuth(
|
||||
'codex-preflight-unknown-exhausted.json',
|
||||
'preflight-unknown-exhausted@example.com',
|
||||
'preflight-unknown-exhausted-token'
|
||||
);
|
||||
writeCodexAuth(
|
||||
'codex-preflight-unknown-fallback.json',
|
||||
'preflight-unknown-fallback@example.com',
|
||||
'preflight-unknown-fallback-token'
|
||||
);
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const accountHeader = new Headers(options?.headers).get('ChatGPT-Account-Id') ?? '';
|
||||
if (accountHeader === 'chatgpt-preflight-unknown-fallback@example.com') {
|
||||
return Promise.resolve(new Response('', { status: 500 }));
|
||||
}
|
||||
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
plan_type: 'pro',
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: 100, reset_after_seconds: 3600 },
|
||||
secondary_window: { used_percent: 100, reset_after_seconds: 604800 },
|
||||
},
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const { preflightCheck } = await import('../../../src/cliproxy/quota-manager');
|
||||
const result = await preflightCheck('codex');
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedFrom).toBe('preflight-unknown-exhausted@example.com');
|
||||
expect(result.accountId).toBe('preflight-unknown-fallback@example.com');
|
||||
expect(getDefaultAccount('codex')?.id).toBe('preflight-unknown-fallback@example.com');
|
||||
expect(getAccount('codex', 'preflight-unknown-exhausted@example.com')?.paused).not.toBe(
|
||||
true
|
||||
);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'codex-preflight-unknown-exhausted.json')
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
tmpDir,
|
||||
'.ccs',
|
||||
'cliproxy',
|
||||
'auth-paused',
|
||||
'codex-preflight-unknown-exhausted.json'
|
||||
)
|
||||
)
|
||||
).toBe(false);
|
||||
expect(fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'quota-paused.json'))).toBe(
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
it('should self-pause exhausted Gemini accounts when a healthy fallback exists', async () => {
|
||||
writeRegistry({
|
||||
gemini: {
|
||||
default: 'gemini-exhausted@example.com',
|
||||
accounts: {
|
||||
'gemini-exhausted@example.com': {
|
||||
email: 'gemini-exhausted@example.com',
|
||||
tokenFile: 'gemini-exhausted.json',
|
||||
},
|
||||
'gemini-fallback@example.com': {
|
||||
email: 'gemini-fallback@example.com',
|
||||
tokenFile: 'gemini-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeGeminiAuth('gemini-exhausted.json', 'gemini-exhausted@example.com', 'exhausted-token');
|
||||
writeGeminiAuth('gemini-fallback.json', 'gemini-fallback@example.com', 'fallback-token');
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
|
||||
const remainingFraction = authHeader === 'Bearer fallback-token' ? 0.9 : 0;
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
buckets: [
|
||||
{
|
||||
model_id: 'gemini-3-flash-preview',
|
||||
remaining_fraction: remainingFraction,
|
||||
remaining_amount: Math.round(remainingFraction * 100),
|
||||
reset_time: '2026-05-01T00:00:00Z',
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await handleQuotaExhaustion('gemini', 'gemini-exhausted@example.com', 10);
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedTo).toBe('gemini-fallback@example.com');
|
||||
expect(getDefaultAccount('gemini')?.id).toBe('gemini-fallback@example.com');
|
||||
expect(getAccount('gemini', 'gemini-exhausted@example.com')?.paused).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'gemini-exhausted.json'))
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth', 'gemini-exhausted.json'))
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('should self-pause exhausted GitHub Copilot accounts when a healthy fallback exists', async () => {
|
||||
writeRegistry({
|
||||
ghcp: {
|
||||
default: 'ghcp-exhausted',
|
||||
accounts: {
|
||||
'ghcp-exhausted': {
|
||||
tokenFile: 'ghcp-exhausted.json',
|
||||
},
|
||||
'ghcp-fallback': {
|
||||
tokenFile: 'ghcp-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeGhcpAuth('ghcp-exhausted.json', 'exhausted-token');
|
||||
writeGhcpAuth('ghcp-fallback.json', 'fallback-token');
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
|
||||
const remaining = authHeader === 'token fallback-token' ? 90 : 0;
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
copilot_plan: 'individual',
|
||||
quota_reset_date: '2026-05-01T00:00:00Z',
|
||||
quota_snapshots: {
|
||||
premium_interactions: { entitlement: 100, remaining },
|
||||
chat: { entitlement: 100, remaining },
|
||||
completions: { entitlement: 100, remaining },
|
||||
},
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await handleQuotaExhaustion('ghcp', 'ghcp-exhausted', 10);
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedTo).toBe('ghcp-fallback');
|
||||
expect(getDefaultAccount('ghcp')?.id).toBe('ghcp-fallback');
|
||||
expect(getAccount('ghcp', 'ghcp-exhausted')?.paused).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'ghcp-exhausted.json'))
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('should ignore omitted GitHub Copilot snapshots when selecting a fallback', async () => {
|
||||
writeRegistry({
|
||||
ghcp: {
|
||||
default: 'ghcp-partial-exhausted',
|
||||
accounts: {
|
||||
'ghcp-partial-exhausted': {
|
||||
tokenFile: 'ghcp-partial-exhausted.json',
|
||||
},
|
||||
'ghcp-partial-fallback': {
|
||||
tokenFile: 'ghcp-partial-fallback.json',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
writeConfig({
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
tier_priority: ['ultra', 'pro', 'free'],
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
preflight_check: true,
|
||||
},
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 10,
|
||||
},
|
||||
});
|
||||
|
||||
writeGhcpAuth('ghcp-partial-exhausted.json', 'partial-exhausted-token');
|
||||
writeGhcpAuth('ghcp-partial-fallback.json', 'partial-fallback-token');
|
||||
|
||||
global.fetch = mock((_url: string, options?: RequestInit) => {
|
||||
const authHeader = new Headers(options?.headers).get('Authorization') ?? '';
|
||||
const remaining = authHeader === 'token partial-fallback-token' ? 90 : 0;
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
copilot_plan: 'individual',
|
||||
quota_reset_date: '2026-05-01T00:00:00Z',
|
||||
quota_snapshots: {
|
||||
premium_interactions: { entitlement: 100, remaining },
|
||||
chat: { entitlement: 100, remaining },
|
||||
},
|
||||
})
|
||||
);
|
||||
}) as typeof fetch;
|
||||
|
||||
const result = await handleQuotaExhaustion('ghcp', 'ghcp-partial-exhausted', 10);
|
||||
const { getAccount, getDefaultAccount } = await import(
|
||||
'../../../src/cliproxy/account-manager'
|
||||
);
|
||||
|
||||
expect(result.switchedTo).toBe('ghcp-partial-fallback');
|
||||
expect(getDefaultAccount('ghcp')?.id).toBe('ghcp-partial-fallback');
|
||||
expect(getAccount('ghcp', 'ghcp-partial-exhausted')?.paused).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(tmpDir, '.ccs', 'cliproxy', 'auth-paused', 'ghcp-partial-exhausted.json')
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ describe('GHCP Quota Fetcher', () => {
|
||||
const snapshot = normalizeGhcpSnapshot();
|
||||
|
||||
expect(snapshot).toEqual({
|
||||
reported: false,
|
||||
entitlement: 0,
|
||||
remaining: 0,
|
||||
used: 0,
|
||||
@@ -88,6 +89,50 @@ describe('GHCP Quota Fetcher', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('marks upstream-provided snapshots as reported', () => {
|
||||
const snapshot = normalizeGhcpSnapshot({
|
||||
entitlement: 100,
|
||||
remaining: 80,
|
||||
});
|
||||
|
||||
expect(snapshot.reported).toBe(true);
|
||||
});
|
||||
|
||||
it('treats unknown upstream snapshot objects as unreported', () => {
|
||||
const empty = normalizeGhcpSnapshot({});
|
||||
const quotaIdOnly = normalizeGhcpSnapshot({ quota_id: 'chat' });
|
||||
const remainingOnly = normalizeGhcpSnapshot({ remaining: 80 });
|
||||
|
||||
for (const snapshot of [empty, quotaIdOnly, remainingOnly]) {
|
||||
expect(snapshot.reported).toBe(false);
|
||||
expect(snapshot.percentRemaining).toBe(0);
|
||||
expect(snapshot.percentUsed).toBe(100);
|
||||
}
|
||||
});
|
||||
|
||||
it('marks percent-only snapshots as reported', () => {
|
||||
const snapshot = normalizeGhcpSnapshot({
|
||||
percent_remaining: 70,
|
||||
});
|
||||
|
||||
expect(snapshot.reported).toBe(true);
|
||||
expect(snapshot.percentRemaining).toBe(70);
|
||||
expect(snapshot.percentUsed).toBe(30);
|
||||
});
|
||||
|
||||
it('treats unlimited snapshots as 100% remaining', () => {
|
||||
const snapshot = normalizeGhcpSnapshot({
|
||||
unlimited: true,
|
||||
entitlement: 0,
|
||||
remaining: 0,
|
||||
});
|
||||
|
||||
expect(snapshot.reported).toBe(true);
|
||||
expect(snapshot.percentRemaining).toBe(100);
|
||||
expect(snapshot.percentUsed).toBe(0);
|
||||
expect(snapshot.unlimited).toBe(true);
|
||||
});
|
||||
|
||||
it('clamps percent_remaining to 0-100 range', () => {
|
||||
const above = normalizeGhcpSnapshot({
|
||||
entitlement: 100,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Tests the quota monitor lifecycle:
|
||||
* - startQuotaMonitor / stopQuotaMonitor behavior
|
||||
* - No-op conditions for non-agy, manual mode, disabled config
|
||||
* - No-op conditions for unsupported providers, manual mode, disabled config
|
||||
* - Idempotent stopQuotaMonitor
|
||||
*/
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
stopQuotaMonitor,
|
||||
clearQuotaCache,
|
||||
} from '../../../src/cliproxy/quota-manager';
|
||||
import { __testExports as executorTestExports } from '../../../src/cliproxy/executor';
|
||||
|
||||
// Setup test isolation
|
||||
let tmpDir: string;
|
||||
@@ -40,11 +41,43 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe('Runtime Quota Monitor', () => {
|
||||
describe('resolveRuntimeQuotaMonitorProviders', () => {
|
||||
it('should monitor the default provider for non-composite sessions', () => {
|
||||
expect(executorTestExports.resolveRuntimeQuotaMonitorProviders('codex', [])).toEqual([
|
||||
'codex',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should monitor every unique managed provider in composite sessions', () => {
|
||||
expect(
|
||||
executorTestExports.resolveRuntimeQuotaMonitorProviders('claude', [
|
||||
'codex',
|
||||
'qwen',
|
||||
'gemini',
|
||||
'codex',
|
||||
])
|
||||
).toEqual(['codex', 'gemini']);
|
||||
});
|
||||
|
||||
it('should skip unsupported non-composite providers', () => {
|
||||
expect(executorTestExports.resolveRuntimeQuotaMonitorProviders('qwen', [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('startQuotaMonitor', () => {
|
||||
it('should accept non-agy provider without throwing', () => {
|
||||
// Non-agy providers should be silently ignored
|
||||
it('should accept all quota-supported providers without throwing', () => {
|
||||
for (const provider of ['agy', 'claude', 'codex', 'gemini', 'ghcp'] as const) {
|
||||
expect(() => {
|
||||
startQuotaMonitor(provider, 'test@gmail.com');
|
||||
stopQuotaMonitor();
|
||||
}).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it('should ignore unsupported providers without throwing', () => {
|
||||
expect(() => {
|
||||
startQuotaMonitor('gemini', 'test@gmail.com');
|
||||
startQuotaMonitor('qwen', 'test@gmail.com');
|
||||
stopQuotaMonitor();
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
@@ -75,6 +108,54 @@ describe('Runtime Quota Monitor', () => {
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
it('should start independent monitors for distinct managed provider accounts', () => {
|
||||
const configDir = path.join(tmpDir, '.ccs', 'config');
|
||||
fs.mkdirSync(configDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(configDir, 'unified-config.json'),
|
||||
JSON.stringify({
|
||||
version: 2,
|
||||
quota_management: {
|
||||
mode: 'auto',
|
||||
runtime_monitor: {
|
||||
enabled: true,
|
||||
normal_interval_seconds: 300,
|
||||
critical_interval_seconds: 60,
|
||||
warn_threshold: 20,
|
||||
exhaustion_threshold: 5,
|
||||
cooldown_minutes: 5,
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const fakeTimer = {
|
||||
unref: () => fakeTimer,
|
||||
ref: () => fakeTimer,
|
||||
hasRef: () => false,
|
||||
refresh: () => fakeTimer,
|
||||
} as unknown as ReturnType<typeof setTimeout>;
|
||||
const observedDelays: Array<number | undefined> = [];
|
||||
|
||||
globalThis.setTimeout = ((handler: TimerHandler, timeout?: number) => {
|
||||
observedDelays.push(timeout);
|
||||
void handler;
|
||||
return fakeTimer;
|
||||
}) as typeof globalThis.setTimeout;
|
||||
|
||||
try {
|
||||
startQuotaMonitor('codex', 'codex@example.com');
|
||||
startQuotaMonitor('gemini', 'gemini@example.com');
|
||||
startQuotaMonitor('codex', 'codex@example.com');
|
||||
} finally {
|
||||
stopQuotaMonitor();
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
}
|
||||
|
||||
expect(observedDelays).toEqual([300_000, 300_000]);
|
||||
});
|
||||
|
||||
it('should be no-op when config missing or no quota_management', () => {
|
||||
// No config file — should not throw
|
||||
expect(() => {
|
||||
|
||||
@@ -465,7 +465,7 @@ export function QuotaTooltipContent({ quota, resetTime }: QuotaTooltipContentPro
|
||||
},
|
||||
{ label: t('quotaTooltip.chat'), snapshot: quota.snapshots.chat },
|
||||
{ label: t('quotaTooltip.completions'), snapshot: quota.snapshots.completions },
|
||||
];
|
||||
].filter(({ snapshot }) => snapshot.reported !== false);
|
||||
const effectiveResetTime = quota.quotaResetDate ?? resetTime;
|
||||
const planLabel = formatPlanLabel(quota.planType);
|
||||
|
||||
|
||||
@@ -862,6 +862,8 @@ export interface GeminiCliQuotaResult {
|
||||
|
||||
/** GitHub Copilot quota snapshot */
|
||||
export interface GhcpQuotaSnapshot {
|
||||
/** False when upstream omitted this quota category */
|
||||
reported?: boolean;
|
||||
/** Total quota allocation for this category */
|
||||
entitlement: number;
|
||||
/** Remaining quota count */
|
||||
|
||||
+4
-5
@@ -694,11 +694,10 @@ export function getGeminiResetTime(buckets: GeminiCliBucket[]): string | null {
|
||||
export function getMinGhcpQuota(snapshots: GhcpQuotaResult['snapshots']): number | null {
|
||||
if (!snapshots) return null;
|
||||
|
||||
const percentages = [
|
||||
snapshots.premiumInteractions.percentRemaining,
|
||||
snapshots.chat.percentRemaining,
|
||||
snapshots.completions.percentRemaining,
|
||||
].filter((p) => typeof p === 'number' && isFinite(p));
|
||||
const percentages = [snapshots.premiumInteractions, snapshots.chat, snapshots.completions]
|
||||
.filter((snapshot) => snapshot.reported !== false)
|
||||
.map((snapshot) => (snapshot.unlimited ? 100 : snapshot.percentRemaining))
|
||||
.filter((p) => typeof p === 'number' && isFinite(p));
|
||||
|
||||
if (percentages.length === 0) return null;
|
||||
return Math.min(...percentages);
|
||||
|
||||
Reference in New Issue
Block a user