mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-16 02:11:28 +00:00
fix(cliproxy): surface gemini quota failure details
- preserve structured HTTP/error metadata for Gemini quota failures - keep upstream verification detail and actionable hints for dashboard users - add regression coverage for 401, 403, and 429 Gemini quota responses
This commit is contained in:
Tam Nhu Tran
@@ -16,6 +16,7 @@ import type { GeminiCliQuotaResult, GeminiCliBucket } from './quota-types';
|
||||
/** Google Cloud Code API endpoints */
|
||||
const GEMINI_CLI_API_BASE = 'https://cloudcode-pa.googleapis.com';
|
||||
const GEMINI_CLI_API_VERSION = 'v1internal';
|
||||
const GEMINI_CLI_ERROR_DETAIL_MAX_LENGTH = 320;
|
||||
|
||||
/**
|
||||
* Model groups for quota consolidation.
|
||||
@@ -68,6 +69,12 @@ interface GeminiCliQuotaResponse {
|
||||
buckets?: RawGeminiCliBucket[];
|
||||
}
|
||||
|
||||
interface ParsedGeminiCliErrorBody {
|
||||
errorCode?: string;
|
||||
errorDetail?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract project ID from account field
|
||||
* Input: "user@example.com (cloudaicompanion-abc-123)"
|
||||
@@ -240,6 +247,213 @@ function shouldIgnoreModel(modelId: string): boolean {
|
||||
return IGNORED_MODEL_PREFIXES.some((prefix) => modelId.startsWith(prefix));
|
||||
}
|
||||
|
||||
function buildGeminiCliFailureResult(
|
||||
accountId: string,
|
||||
projectId: string | null,
|
||||
options: {
|
||||
error: string;
|
||||
httpStatus?: number;
|
||||
errorCode?: string;
|
||||
errorDetail?: string;
|
||||
actionHint?: string;
|
||||
retryable?: boolean;
|
||||
needsReauth?: boolean;
|
||||
isForbidden?: boolean;
|
||||
}
|
||||
): GeminiCliQuotaResult {
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId,
|
||||
lastUpdated: Date.now(),
|
||||
accountId,
|
||||
error: options.error,
|
||||
httpStatus: options.httpStatus,
|
||||
errorCode: options.errorCode,
|
||||
errorDetail: options.errorDetail,
|
||||
actionHint: options.actionHint,
|
||||
retryable: options.retryable,
|
||||
needsReauth: options.needsReauth,
|
||||
isForbidden: options.isForbidden,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeGeminiCliErrorDetail(bodyText: string): string | undefined {
|
||||
const trimmed = bodyText.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (/^<!doctype html/i.test(trimmed) || /^<html/i.test(trimmed) || /^<[^>]+>/.test(trimmed)) {
|
||||
return '[HTML error response omitted]';
|
||||
}
|
||||
|
||||
let sanitized = trimmed
|
||||
.replace(
|
||||
/"(access[_-]?token|refresh[_-]?token|authorization|cookie|set-cookie|api[_-]?key|session[_-]?token|token)"\s*:\s*"[^"]*"/gi,
|
||||
'"$1":"[redacted]"'
|
||||
)
|
||||
.replace(/Bearer\s+[A-Za-z0-9._-]+/g, 'Bearer [redacted]')
|
||||
.replace(/\s+/g, ' ');
|
||||
|
||||
if (sanitized.length > GEMINI_CLI_ERROR_DETAIL_MAX_LENGTH) {
|
||||
sanitized = `${sanitized.slice(0, GEMINI_CLI_ERROR_DETAIL_MAX_LENGTH - 14)}...[truncated]`;
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function extractGeminiCliNestedMessage(value: unknown): string | undefined {
|
||||
if (Array.isArray(value)) {
|
||||
for (const entry of value) {
|
||||
const nested = extractGeminiCliNestedMessage(entry);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!value || typeof value !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const record = value as Record<string, unknown>;
|
||||
const directMessage = [
|
||||
record.message,
|
||||
record.localizedMessage,
|
||||
record.description,
|
||||
record.reason,
|
||||
record.error,
|
||||
].find((candidate): candidate is string => typeof candidate === 'string' && candidate.trim().length > 0);
|
||||
if (directMessage) {
|
||||
return directMessage;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseGeminiCliErrorBody(bodyText: string): ParsedGeminiCliErrorBody {
|
||||
const trimmed = bodyText.trim();
|
||||
if (!trimmed) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const sanitizedDetail = sanitizeGeminiCliErrorDetail(trimmed);
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
const topLevelMessage = [parsed.message, parsed.error]
|
||||
.find((candidate): candidate is string => typeof candidate === 'string' && candidate.trim().length > 0);
|
||||
const topLevelCode = [parsed.code, parsed.status]
|
||||
.find((candidate): candidate is string => typeof candidate === 'string' && candidate.trim().length > 0);
|
||||
|
||||
if (parsed.error && typeof parsed.error === 'object') {
|
||||
const error = parsed.error as Record<string, unknown>;
|
||||
return {
|
||||
errorCode:
|
||||
[error.status, error.code, topLevelCode].find(
|
||||
(candidate): candidate is string =>
|
||||
typeof candidate === 'string' && candidate.trim().length > 0
|
||||
) || undefined,
|
||||
errorDetail: sanitizedDetail,
|
||||
message:
|
||||
[error.message, error.error, extractGeminiCliNestedMessage(error.details), topLevelMessage].find(
|
||||
(candidate): candidate is string =>
|
||||
typeof candidate === 'string' && candidate.trim().length > 0
|
||||
) || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
errorCode: topLevelCode,
|
||||
errorDetail: sanitizedDetail,
|
||||
message:
|
||||
[topLevelMessage, extractGeminiCliNestedMessage(parsed.details)].find(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.trim().length > 0
|
||||
) || undefined,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
errorDetail: sanitizedDetail,
|
||||
message: trimmed,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function buildGeminiCliForbiddenActionHint(parsed: ParsedGeminiCliErrorBody): string {
|
||||
const combined = `${parsed.message || ''} ${parsed.errorDetail || ''}`.toLowerCase();
|
||||
if (combined.includes('verify') || combined.includes('verification')) {
|
||||
return 'Complete the Google account verification mentioned above, then retry quota refresh.';
|
||||
}
|
||||
if (combined.includes('project')) {
|
||||
return 'Confirm this Google project still has Gemini CLI quota access, then retry.';
|
||||
}
|
||||
return 'Check the Google account or workspace access shown above, then retry quota refresh.';
|
||||
}
|
||||
|
||||
function buildGeminiCliHttpFailureResult(
|
||||
accountId: string,
|
||||
projectId: string | null,
|
||||
status: number,
|
||||
bodyText: string
|
||||
): GeminiCliQuotaResult {
|
||||
const parsed = parseGeminiCliErrorBody(bodyText);
|
||||
|
||||
if (status === 401) {
|
||||
return buildGeminiCliFailureResult(accountId, projectId, {
|
||||
error: parsed.message || 'Token expired or invalid',
|
||||
httpStatus: 401,
|
||||
errorCode: parsed.errorCode || 'reauth_required',
|
||||
errorDetail: parsed.errorDetail,
|
||||
actionHint: 'Run ccs gemini --auth to reconnect this account.',
|
||||
needsReauth: true,
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (status === 403) {
|
||||
return buildGeminiCliFailureResult(accountId, projectId, {
|
||||
error: parsed.message || 'Quota access forbidden for this account',
|
||||
httpStatus: 403,
|
||||
errorCode: parsed.errorCode || 'quota_api_forbidden',
|
||||
errorDetail: parsed.errorDetail,
|
||||
actionHint: buildGeminiCliForbiddenActionHint(parsed),
|
||||
isForbidden: true,
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (status === 429) {
|
||||
return buildGeminiCliFailureResult(accountId, projectId, {
|
||||
error: parsed.message || 'Rate limited - try again later',
|
||||
httpStatus: 429,
|
||||
errorCode: parsed.errorCode || 'rate_limited',
|
||||
errorDetail: parsed.errorDetail,
|
||||
actionHint: 'Retry after a short delay.',
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (status >= 500) {
|
||||
return buildGeminiCliFailureResult(accountId, projectId, {
|
||||
error: parsed.message || `Gemini quota service unavailable (HTTP ${status})`,
|
||||
httpStatus: status,
|
||||
errorCode: parsed.errorCode || 'provider_unavailable',
|
||||
errorDetail: parsed.errorDetail,
|
||||
actionHint: 'Retry later. This looks like a temporary Google upstream problem.',
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
return buildGeminiCliFailureResult(accountId, projectId, {
|
||||
error: parsed.message || `Gemini quota request failed (HTTP ${status})`,
|
||||
httpStatus: status,
|
||||
errorCode: parsed.errorCode || 'quota_request_failed',
|
||||
errorDetail: parsed.errorDetail,
|
||||
actionHint: 'Inspect the upstream response details and retry if appropriate.',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build GeminiCliBucket array from API response
|
||||
* Groups buckets by model series and token type
|
||||
@@ -334,14 +548,12 @@ async function fetchWithAuthData(
|
||||
if (!authData.projectId) {
|
||||
const error = 'Cannot resolve project ID from auth file';
|
||||
if (verbose) console.error(`[!] Error: ${error}`);
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: null,
|
||||
lastUpdated: Date.now(),
|
||||
return buildGeminiCliFailureResult(accountId, null, {
|
||||
error,
|
||||
accountId,
|
||||
};
|
||||
errorCode: 'missing_project_id',
|
||||
actionHint: 'Run ccs gemini --auth to reconnect this account and recover the project ID.',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
const url = `${GEMINI_CLI_API_BASE}/${GEMINI_CLI_API_VERSION}:retrieveUserQuota`;
|
||||
@@ -363,49 +575,9 @@ async function fetchWithAuthData(
|
||||
|
||||
if (verbose) console.error(`[i] Gemini CLI API status: ${response.status}`);
|
||||
|
||||
if (response.status === 401) {
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: authData.projectId,
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Token expired or invalid',
|
||||
accountId,
|
||||
needsReauth: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (response.status === 403) {
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: authData.projectId,
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Quota access forbidden for this account',
|
||||
accountId,
|
||||
};
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: authData.projectId,
|
||||
lastUpdated: Date.now(),
|
||||
error: 'Rate limited - try again later',
|
||||
accountId,
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: authData.projectId,
|
||||
lastUpdated: Date.now(),
|
||||
error: `API error: ${response.status}`,
|
||||
accountId,
|
||||
};
|
||||
const bodyText = await response.text();
|
||||
return buildGeminiCliHttpFailureResult(accountId, authData.projectId, response.status, bodyText);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as GeminiCliQuotaResponse;
|
||||
@@ -432,14 +604,13 @@ async function fetchWithAuthData(
|
||||
|
||||
if (verbose) console.error(`[!] Gemini CLI quota error: ${errorMsg}`);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: authData.projectId,
|
||||
lastUpdated: Date.now(),
|
||||
return buildGeminiCliFailureResult(accountId, authData.projectId, {
|
||||
error: errorMsg,
|
||||
accountId,
|
||||
};
|
||||
errorCode: err instanceof Error && err.name === 'AbortError' ? 'network_timeout' : 'network_error',
|
||||
actionHint: 'Retry later. This looks temporary.',
|
||||
retryable: true,
|
||||
httpStatus: err instanceof Error && err.name === 'AbortError' ? 408 : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,14 +631,12 @@ export async function fetchGeminiCliQuota(
|
||||
if (!authData) {
|
||||
const error = 'Auth file not found for Gemini account';
|
||||
if (verbose) console.error(`[!] Error: ${error}`);
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: null,
|
||||
lastUpdated: Date.now(),
|
||||
return buildGeminiCliFailureResult(accountId, null, {
|
||||
error,
|
||||
accountId,
|
||||
};
|
||||
errorCode: 'auth_file_missing',
|
||||
actionHint: 'Run ccs gemini --auth to reconnect this account.',
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
// Proactive refresh: refresh if expired OR expiring within 5 minutes
|
||||
@@ -497,15 +666,14 @@ export async function fetchGeminiCliQuota(
|
||||
// Only fail if token is actually expired (not just expiring soon)
|
||||
const error = refreshResult.error || 'Token refresh failed';
|
||||
if (verbose) console.error(`[!] Refresh failed: ${error}`);
|
||||
return {
|
||||
success: false,
|
||||
buckets: [],
|
||||
projectId: null,
|
||||
lastUpdated: Date.now(),
|
||||
return buildGeminiCliFailureResult(accountId, authData.projectId, {
|
||||
error,
|
||||
accountId,
|
||||
errorCode: 'reauth_required',
|
||||
errorDetail: error,
|
||||
actionHint: 'Run ccs gemini --auth to reconnect this account.',
|
||||
needsReauth: true,
|
||||
};
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
// If proactive refresh fails but token isn't expired yet, continue with existing token
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
||||
import { getCapturedFetchRequests, mockFetch, restoreFetch } from '../../mocks';
|
||||
|
||||
describe('Gemini CLI Quota Fetcher', () => {
|
||||
const GEMINI_QUOTA_URL = 'https://cloudcode-pa.googleapis.com/v1internal:retrieveUserQuota';
|
||||
const GOOGLE_TOKEN_URL = 'https://oauth2.googleapis.com/token';
|
||||
let tempHome: string;
|
||||
let originalCcsHome: string | undefined;
|
||||
let originalCcsDir: string | undefined;
|
||||
@@ -18,6 +20,7 @@ describe('Gemini CLI Quota Fetcher', () => {
|
||||
let originalGeminiClientSecret: string | undefined;
|
||||
let moduleVersion = 0;
|
||||
let buildGeminiCliBuckets: typeof import('../../../src/cliproxy/quota-fetcher-gemini-cli').buildGeminiCliBuckets;
|
||||
let fetchGeminiCliQuota: typeof import('../../../src/cliproxy/quota-fetcher-gemini-cli').fetchGeminiCliQuota;
|
||||
let resolveGeminiCliProjectId: typeof import('../../../src/cliproxy/quota-fetcher-gemini-cli').resolveGeminiCliProjectId;
|
||||
let refreshGeminiToken: typeof import('../../../src/cliproxy/auth/gemini-token-refresh').refreshGeminiToken;
|
||||
let getProviderAuthDir: typeof import('../../../src/cliproxy/config-generator').getProviderAuthDir;
|
||||
@@ -30,6 +33,23 @@ describe('Gemini CLI Quota Fetcher', () => {
|
||||
return tokenPath;
|
||||
}
|
||||
|
||||
function writeActiveGeminiAccount(accountId: string, overrides: Record<string, unknown> = {}): string {
|
||||
return writeGeminiToken({
|
||||
type: 'gemini',
|
||||
email: accountId,
|
||||
project_id: 'cloudaicompanion-test-123',
|
||||
token: {
|
||||
access_token: 'access-token',
|
||||
refresh_token: 'refresh-token',
|
||||
expiry: Date.now() + 60 * 60 * 1000,
|
||||
client_id: 'test-client-id',
|
||||
client_secret: 'test-client-secret',
|
||||
token_uri: GOOGLE_TOKEN_URL,
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
moduleVersion += 1;
|
||||
tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-gemini-refresh-'));
|
||||
@@ -46,7 +66,7 @@ describe('Gemini CLI Quota Fetcher', () => {
|
||||
const configGenerator = await import(
|
||||
`../../../src/cliproxy/config-generator?gemini-config-generator=${moduleVersion}`
|
||||
);
|
||||
({ buildGeminiCliBuckets, resolveGeminiCliProjectId } = await import(
|
||||
({ buildGeminiCliBuckets, fetchGeminiCliQuota, resolveGeminiCliProjectId } = await import(
|
||||
`../../../src/cliproxy/quota-fetcher-gemini-cli?gemini-quota-fetcher=${moduleVersion}`
|
||||
));
|
||||
({ refreshGeminiToken } = await import(
|
||||
@@ -265,6 +285,104 @@ describe('Gemini CLI Quota Fetcher', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchGeminiCliQuota failure metadata', () => {
|
||||
it('maps 401 responses to reauth-required metadata', async () => {
|
||||
writeActiveGeminiAccount('reauth@example.com');
|
||||
|
||||
mockFetch([
|
||||
{
|
||||
url: GEMINI_QUOTA_URL,
|
||||
method: 'POST',
|
||||
status: 401,
|
||||
response: {
|
||||
error: {
|
||||
message: 'Session expired',
|
||||
status: 'UNAUTHENTICATED',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
url: GOOGLE_TOKEN_URL,
|
||||
method: 'POST',
|
||||
status: 400,
|
||||
response: {
|
||||
error: 'invalid_grant',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await fetchGeminiCliQuota('reauth@example.com');
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.httpStatus).toBe(401);
|
||||
expect(result.errorCode).toBe('UNAUTHENTICATED');
|
||||
expect(result.needsReauth).toBe(true);
|
||||
expect(result.retryable).toBe(false);
|
||||
expect(result.actionHint).toContain('ccs gemini --auth');
|
||||
expect(result.error).toBe('Session expired');
|
||||
});
|
||||
|
||||
it('preserves 403 verification detail and exposes a helpful action hint', async () => {
|
||||
writeActiveGeminiAccount('verify@example.com');
|
||||
|
||||
mockFetch([
|
||||
{
|
||||
url: GEMINI_QUOTA_URL,
|
||||
method: 'POST',
|
||||
status: 403,
|
||||
response: {
|
||||
error: {
|
||||
message: 'Google requires you to verify this account before using Gemini CLI quota.',
|
||||
status: 'PERMISSION_DENIED',
|
||||
details: [
|
||||
{
|
||||
reason: 'ACCOUNT_VERIFICATION_REQUIRED',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await fetchGeminiCliQuota('verify@example.com');
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.httpStatus).toBe(403);
|
||||
expect(result.isForbidden).toBe(true);
|
||||
expect(result.retryable).toBe(false);
|
||||
expect(result.error).toContain('verify this account');
|
||||
expect(result.actionHint).toContain('verification');
|
||||
expect(result.errorDetail).toContain('ACCOUNT_VERIFICATION_REQUIRED');
|
||||
});
|
||||
|
||||
it('marks 429 responses as retryable', async () => {
|
||||
writeActiveGeminiAccount('rate-limit@example.com');
|
||||
|
||||
mockFetch([
|
||||
{
|
||||
url: GEMINI_QUOTA_URL,
|
||||
method: 'POST',
|
||||
status: 429,
|
||||
response: {
|
||||
error: {
|
||||
message: 'Too many quota requests',
|
||||
status: 'RESOURCE_EXHAUSTED',
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await fetchGeminiCliQuota('rate-limit@example.com');
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.httpStatus).toBe(429);
|
||||
expect(result.retryable).toBe(true);
|
||||
expect(result.errorCode).toBe('RESOURCE_EXHAUSTED');
|
||||
expect(result.actionHint).toContain('Retry');
|
||||
expect(result.error).toBe('Too many quota requests');
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshGeminiToken', () => {
|
||||
it('uses OAuth client metadata stored in the token file', async () => {
|
||||
writeGeminiToken({
|
||||
|
||||
Reference in New Issue
Block a user