refactor(cliproxy): enhance binary downloader with robust error handling

- Add categorized error detection for socket, timeout, HTTP, and redirect errors
- Implement smarter exponential backoff (longer delays for socket errors)
- Increase max retries from 3 to 5 with dynamic timeout adjustment
- Add resource cleanup (disable connection pooling for clean process exit)
- Add user-friendly error messages for each error type
- Refactor fetchText/fetchJson with retry logic and proper error handling
- Set 120s timeout for large binary downloads, 15-30s for text/JSON
- Prevent double-resolution and race conditions in Promise handlers
This commit is contained in:
kaitranntt
2025-12-22 18:22:32 -05:00
parent c367149756
commit c2dd0261b7
3 changed files with 285 additions and 51 deletions
+261 -44
View File
@@ -1,6 +1,7 @@
/** /**
* Binary Downloader * Binary Downloader
* Handles downloading files with retry logic, progress tracking, and redirect following. * Handles downloading files with retry logic, progress tracking, and redirect following.
* Robust handling for transient network errors (socket hang up, ECONNRESET, etc.)
*/ */
import * as fs from 'fs'; import * as fs from 'fs';
@@ -14,40 +15,111 @@ export interface DownloaderConfig {
maxRetries: number; maxRetries: number;
/** Enable verbose logging */ /** Enable verbose logging */
verbose: boolean; verbose: boolean;
/** Timeout in milliseconds (default: 120000 for large files) */
timeout?: number;
} }
const DEFAULT_CONFIG: DownloaderConfig = { const DEFAULT_CONFIG: DownloaderConfig = {
maxRetries: 3, maxRetries: 5,
verbose: false, verbose: false,
timeout: 120000, // 2 minutes for large binaries
}; };
/** Error types for categorized handling */
export type NetworkErrorType = 'socket' | 'timeout' | 'http' | 'redirect' | 'unknown';
/** Categorize error for appropriate retry/reporting */
export function categorizeError(error: Error): NetworkErrorType {
const msg = error.message.toLowerCase();
if (msg.includes('socket hang up') || msg.includes('econnreset') || msg.includes('epipe')) {
return 'socket';
}
if (msg.includes('timeout') || msg.includes('etimedout')) {
return 'timeout';
}
if (msg.includes('http ')) {
return 'http';
}
if (msg.includes('redirect')) {
return 'redirect';
}
return 'unknown';
}
/** Get user-friendly error message */
export function getErrorMessage(error: Error, attempt: number, maxAttempts: number): string {
const type = categorizeError(error);
const prefix = `[Attempt ${attempt}/${maxAttempts}]`;
switch (type) {
case 'socket':
return `${prefix} Connection dropped (socket hang up) - retrying with fresh connection...`;
case 'timeout':
return `${prefix} Download timed out - retrying with extended timeout...`;
case 'http':
return `${prefix} Server error: ${error.message}`;
case 'redirect':
return `${prefix} Redirect failed: ${error.message}`;
default:
return `${prefix} Network error: ${error.message}`;
}
}
/** Check if error is retryable */
export function isRetryableError(error: Error): boolean {
const type = categorizeError(error);
// Retry socket errors, timeouts, and unknown errors
if (type === 'socket' || type === 'timeout' || type === 'unknown') {
return true;
}
if (type === 'http') {
const msg = error.message;
// Retry 5xx server errors and 429 rate limit (HTTP 5xx matches 500, 502, 503, 504, etc.)
return msg.includes('HTTP 5') || msg.includes('HTTP 429');
}
return false;
}
/** /**
* Download file from URL with progress tracking * Download file from URL with progress tracking
* @param timeout Timeout in ms (default 120000 for large files)
*/ */
export function downloadFile( export function downloadFile(
url: string, url: string,
destPath: string, destPath: string,
onProgress?: ProgressCallback, onProgress?: ProgressCallback,
verbose = false verbose = false,
timeout = 120000
): Promise<void> { ): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let resolved = false;
const cleanup = (err?: Error) => {
if (resolved) return;
resolved = true;
fs.unlink(destPath, () => {}); // Cleanup partial file
reject(err || new Error('Download aborted'));
};
const handleResponse = (res: http.IncomingMessage) => { const handleResponse = (res: http.IncomingMessage) => {
// Handle redirects (GitHub releases use 302) // Handle redirects (GitHub releases use 302)
if (res.statusCode === 301 || res.statusCode === 302) { if (res.statusCode === 301 || res.statusCode === 302) {
const redirectUrl = res.headers.location; const redirectUrl = res.headers.location;
if (!redirectUrl) { if (!redirectUrl) {
reject(new Error('Redirect without location header')); cleanup(new Error('Redirect without location header'));
return; return;
} }
if (verbose) { if (verbose) {
console.error(`[cliproxy] Following redirect: ${redirectUrl}`); console.error(`[cliproxy] Following redirect: ${redirectUrl}`);
} }
downloadFile(redirectUrl, destPath, onProgress, verbose).then(resolve).catch(reject); downloadFile(redirectUrl, destPath, onProgress, verbose, timeout)
.then(resolve)
.catch(reject);
return; return;
} }
if (res.statusCode !== 200) { if (res.statusCode !== 200) {
reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); cleanup(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
return; return;
} }
@@ -70,58 +142,94 @@ export function downloadFile(
res.pipe(fileStream); res.pipe(fileStream);
fileStream.on('finish', () => { fileStream.on('finish', () => {
if (resolved) return;
resolved = true;
fileStream.close(); fileStream.close();
resolve(); resolve();
}); });
fileStream.on('error', (err) => { fileStream.on('error', cleanup);
fs.unlink(destPath, () => {}); // Cleanup partial file res.on('error', cleanup);
reject(err);
});
res.on('error', (err) => {
fs.unlink(destPath, () => {});
reject(err);
});
}; };
const protocol = url.startsWith('https') ? https : http; const protocol = url.startsWith('https') ? https : http;
const req = protocol.get(url, handleResponse);
req.on('error', reject); // Use agent: false to prevent connection pooling (allows process to exit)
req.setTimeout(60000, () => { const options = {
headers: {
'User-Agent': 'CCS-CLIProxyPlus-Downloader/1.0',
},
agent: false, // Disable connection pooling for clean exit
};
const req = protocol.get(url, options, handleResponse);
req.on('error', (err) => {
if (!resolved) {
cleanup(err);
}
});
req.setTimeout(timeout, () => {
req.destroy(); req.destroy();
reject(new Error('Download timeout (60s)')); if (!resolved) {
cleanup(new Error(`Download timeout (${timeout / 1000}s)`));
}
}); });
}); });
} }
/** /**
* Download file with retry logic and exponential backoff * Download file with retry logic and exponential backoff
* Uses smarter backoff for socket errors (longer delays)
*/ */
export async function downloadWithRetry( export async function downloadWithRetry(
url: string, url: string,
destPath: string, destPath: string,
config: Partial<DownloaderConfig> = {} config: Partial<DownloaderConfig> = {}
): Promise<DownloadResult> { ): Promise<DownloadResult> {
const { maxRetries, verbose } = { ...DEFAULT_CONFIG, ...config }; const { maxRetries, verbose, timeout } = { ...DEFAULT_CONFIG, ...config };
let lastError = ''; let lastError: Error | null = null;
let retries = 0; let retries = 0;
let currentTimeout = timeout || 120000;
while (retries < maxRetries) { while (retries < maxRetries) {
try { try {
await downloadFile(url, destPath, undefined, verbose); await downloadFile(url, destPath, undefined, verbose, currentTimeout);
return { success: true, filePath: destPath, retries }; return { success: true, filePath: destPath, retries };
} catch (error) { } catch (error) {
const err = error as Error; const err = error as Error;
lastError = err.message; lastError = err;
retries++; retries++;
if (retries < maxRetries) { // Check if error is retryable
// Exponential backoff: 1s, 2s, 4s if (!isRetryableError(err)) {
const delay = Math.pow(2, retries - 1) * 1000;
if (verbose) { if (verbose) {
console.error(`[cliproxy] Retry ${retries}/${maxRetries} after ${delay}ms: ${lastError}`); console.error(`[cliproxy] Non-retryable error: ${err.message}`);
}
break;
}
if (retries < maxRetries) {
const errorType = categorizeError(err);
// Socket errors: longer backoff (2s, 4s, 8s, 16s, 32s)
// Timeout errors: increase timeout and use standard backoff
// Other errors: standard exponential backoff (1s, 2s, 4s...)
let delay: number;
if (errorType === 'socket') {
delay = Math.pow(2, retries) * 1000; // 2s, 4s, 8s, 16s, 32s
} else if (errorType === 'timeout') {
delay = Math.pow(2, retries - 1) * 1000;
currentTimeout = Math.min(currentTimeout * 1.5, 300000); // Increase timeout up to 5 min
} else {
delay = Math.pow(2, retries - 1) * 1000;
}
// Log with user-friendly message
console.error(`[cliproxy] ${getErrorMessage(err, retries, maxRetries)}`);
if (verbose) {
console.error(`[cliproxy] Waiting ${delay}ms before retry...`);
} }
await sleep(delay); await sleep(delay);
} }
@@ -130,16 +238,18 @@ export async function downloadWithRetry(
return { return {
success: false, success: false,
error: `Download failed after ${retries} attempts: ${lastError}`, error: `Download failed after ${retries} attempts: ${lastError?.message || 'Unknown error'}`,
retries, retries,
}; };
} }
/** /**
* Fetch text content from URL * Fetch text content from URL (single attempt)
*/ */
export function fetchText(url: string, verbose = false): Promise<string> { function fetchTextOnce(url: string, verbose = false, timeout = 30000): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let resolved = false;
const handleResponse = (res: http.IncomingMessage) => { const handleResponse = (res: http.IncomingMessage) => {
// Handle redirects // Handle redirects
if (res.statusCode === 301 || res.statusCode === 302) { if (res.statusCode === 301 || res.statusCode === 302) {
@@ -148,7 +258,7 @@ export function fetchText(url: string, verbose = false): Promise<string> {
reject(new Error('Redirect without location header')); reject(new Error('Redirect without location header'));
return; return;
} }
fetchText(redirectUrl, verbose).then(resolve).catch(reject); fetchTextOnce(redirectUrl, verbose, timeout).then(resolve).catch(reject);
return; return;
} }
@@ -159,30 +269,90 @@ export function fetchText(url: string, verbose = false): Promise<string> {
let data = ''; let data = '';
res.on('data', (chunk) => (data += chunk)); res.on('data', (chunk) => (data += chunk));
res.on('end', () => resolve(data)); res.on('end', () => {
res.on('error', reject); if (!resolved) {
resolved = true;
resolve(data);
}
});
res.on('error', (err) => {
if (!resolved) {
resolved = true;
reject(err);
}
});
}; };
const protocol = url.startsWith('https') ? https : http; const protocol = url.startsWith('https') ? https : http;
const req = protocol.get(url, handleResponse); const options = {
req.on('error', reject); headers: {
req.setTimeout(30000, () => { 'User-Agent': 'CCS-CLIProxyPlus-Downloader/1.0',
},
agent: false, // Disable connection pooling for clean exit
};
const req = protocol.get(url, options, handleResponse);
req.on('error', (err) => {
if (!resolved) {
resolved = true;
reject(err);
}
});
req.setTimeout(timeout, () => {
req.destroy(); req.destroy();
reject(new Error('Request timeout (30s)')); if (!resolved) {
resolved = true;
reject(new Error(`Request timeout (${timeout / 1000}s)`));
}
}); });
}); });
} }
/** /**
* Fetch JSON from URL (for GitHub API) * Fetch text content from URL with retry logic
*/ */
export function fetchJson(url: string, verbose = false): Promise<Record<string, unknown>> { export async function fetchText(url: string, verbose = false, maxRetries = 3): Promise<string> {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
return await fetchTextOnce(url, verbose);
} catch (error) {
const err = error as Error;
lastError = err;
if (!isRetryableError(err) || attempt === maxRetries) {
break;
}
const delay = Math.pow(2, attempt - 1) * 1000;
if (verbose) {
console.error(`[cliproxy] fetchText retry ${attempt}/${maxRetries}: ${err.message}`);
}
await sleep(delay);
}
}
throw lastError || new Error('fetchText failed');
}
/**
* Fetch JSON from URL (single attempt, for GitHub API)
*/
function fetchJsonOnce(
url: string,
verbose = false,
timeout = 15000
): Promise<Record<string, unknown>> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const options = { let resolved = false;
const options: https.RequestOptions = {
headers: { headers: {
'User-Agent': 'CCS-CLIProxyPlus-Updater/1.0', 'User-Agent': 'CCS-CLIProxyPlus-Updater/1.0',
Accept: 'application/vnd.github.v3+json', Accept: 'application/vnd.github.v3+json',
}, },
agent: false, // Disable connection pooling for clean exit
}; };
const handleResponse = (res: http.IncomingMessage) => { const handleResponse = (res: http.IncomingMessage) => {
@@ -192,7 +362,7 @@ export function fetchJson(url: string, verbose = false): Promise<Record<string,
reject(new Error('Redirect without location header')); reject(new Error('Redirect without location header'));
return; return;
} }
fetchJson(redirectUrl, verbose).then(resolve).catch(reject); fetchJsonOnce(redirectUrl, verbose, timeout).then(resolve).catch(reject);
return; return;
} }
@@ -204,24 +374,71 @@ export function fetchJson(url: string, verbose = false): Promise<Record<string,
let data = ''; let data = '';
res.on('data', (chunk) => (data += chunk)); res.on('data', (chunk) => (data += chunk));
res.on('end', () => { res.on('end', () => {
if (resolved) return;
resolved = true;
try { try {
resolve(JSON.parse(data)); resolve(JSON.parse(data));
} catch { } catch {
reject(new Error('Invalid JSON from GitHub API')); reject(new Error('Invalid JSON from GitHub API'));
} }
}); });
res.on('error', reject); res.on('error', (err) => {
if (!resolved) {
resolved = true;
reject(err);
}
});
}; };
const req = https.get(url, options, handleResponse); const req = https.get(url, options, handleResponse);
req.on('error', reject); req.on('error', (err) => {
req.setTimeout(10000, () => { if (!resolved) {
resolved = true;
reject(err);
}
});
req.setTimeout(timeout, () => {
req.destroy(); req.destroy();
reject(new Error('GitHub API timeout (10s)')); if (!resolved) {
resolved = true;
reject(new Error(`GitHub API timeout (${timeout / 1000}s)`));
}
}); });
}); });
} }
/**
* Fetch JSON from URL (for GitHub API) with retry logic
*/
export async function fetchJson(
url: string,
verbose = false,
maxRetries = 3
): Promise<Record<string, unknown>> {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
return await fetchJsonOnce(url, verbose);
} catch (error) {
const err = error as Error;
lastError = err;
if (!isRetryableError(err) || attempt === maxRetries) {
break;
}
const delay = Math.pow(2, attempt - 1) * 1000;
if (verbose) {
console.error(`[cliproxy] GitHub API retry ${attempt}/${maxRetries}: ${err.message}`);
}
await sleep(delay);
}
}
throw lastError || new Error('fetchJson failed');
}
/** /**
* Sleep helper * Sleep helper
*/ */
+9 -5
View File
@@ -424,13 +424,17 @@ export async function execClaudeWithCLIProxy(
if (proxyStatus.running && proxyStatus.verified) { if (proxyStatus.running && proxyStatus.verified) {
// Healthy proxy found - join it // Healthy proxy found - join it
if (proxyStatus.pid) { if (proxyStatus.pid) {
sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.pid) ?? undefined; sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.pid, verbose) ?? undefined;
} }
if (sessionId) { if (sessionId) {
console.log(info(`Joined existing CLIProxy on port ${cfg.port} (${proxyStatus.method})`)); console.log(info(`Joined existing CLIProxy on port ${cfg.port} (${proxyStatus.method})`));
} else { } else {
// Failed to register session, but proxy is running - continue anyway // Failed to register session - proxy is running but we can't track it
log(`Failed to register session but proxy is healthy, continuing...`); // This happens when port-process detection fails (permissions, platform issues)
console.log(
info(`Using existing CLIProxy on port ${cfg.port} (session tracking unavailable)`)
);
log(`PID=${proxyStatus.pid ?? 'unknown'}, session registration skipped`);
} }
return; // Exit lock early, skip spawning return; // Exit lock early, skip spawning
} }
@@ -441,7 +445,7 @@ export async function execClaudeWithCLIProxy(
const becameHealthy = await waitForProxyHealthy(cfg.port, cfg.timeout); const becameHealthy = await waitForProxyHealthy(cfg.port, cfg.timeout);
if (becameHealthy) { if (becameHealthy) {
if (proxyStatus.pid) { if (proxyStatus.pid) {
sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.pid) ?? undefined; sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.pid, verbose) ?? undefined;
} }
console.log(info(`Joined CLIProxy after startup wait`)); console.log(info(`Joined CLIProxy after startup wait`));
return; // Exit lock early return; // Exit lock early
@@ -459,7 +463,7 @@ export async function execClaudeWithCLIProxy(
// Last resort: try HTTP health check (handles Windows PID-XXXXX case) // Last resort: try HTTP health check (handles Windows PID-XXXXX case)
const isActuallyOurs = await waitForProxyHealthy(cfg.port, 1000); const isActuallyOurs = await waitForProxyHealthy(cfg.port, 1000);
if (isActuallyOurs) { if (isActuallyOurs) {
sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.blocker.pid) ?? undefined; sessionId = reclaimOrphanedProxy(cfg.port, proxyStatus.blocker.pid, verbose) ?? undefined;
console.log(info(`Reclaimed CLIProxy with unrecognized process name`)); console.log(info(`Reclaimed CLIProxy with unrecognized process name`));
return; return;
} }
+15 -2
View File
@@ -73,12 +73,25 @@ export async function detectRunningProxy(
// Proxy is running and responsive // Proxy is running and responsive
// Try to get PID from session lock if available // Try to get PID from session lock if available
const lock = getExistingProxy(port); const lock = getExistingProxy(port);
log(`HTTP check passed, proxy healthy (PID: ${lock?.pid ?? 'unknown'})`); let pid = lock?.pid;
// If no PID from session lock, try port-process detection
// This handles orphaned proxies (running but no sessions.json)
if (!pid) {
log('No PID from session lock, checking port process...');
const portProcess = await getPortProcess(port);
if (portProcess) {
pid = portProcess.pid;
log(`Got PID from port process: ${pid}`);
}
}
log(`HTTP check passed, proxy healthy (PID: ${pid ?? 'unknown'})`);
return { return {
running: true, running: true,
verified: true, verified: true,
method: 'http', method: 'http',
pid: lock?.pid, pid,
sessionCount: lock?.sessions?.length, sessionCount: lock?.sessions?.length,
}; };
} }