diff --git a/src/errors/__tests__/error-types.test.ts b/src/errors/__tests__/error-types.test.ts new file mode 100644 index 00000000..0a28b240 --- /dev/null +++ b/src/errors/__tests__/error-types.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect } from 'bun:test'; +import { CCSError, RetryableError, isCCSError, isRecoverableError } from '../error-types'; +import { ExitCode } from '../exit-codes'; + +describe('RetryableError', () => { + it('extends CCSError', () => { + const err = new RetryableError('test'); + expect(err).toBeInstanceOf(CCSError); + expect(err).toBeInstanceOf(RetryableError); + }); + + it('sets name to RetryableError', () => { + const err = new RetryableError('test'); + expect(err.name).toBe('RetryableError'); + }); + + it('sets recoverable to true', () => { + const err = new RetryableError('test'); + expect(err.recoverable).toBe(true); + }); + + it('defaults exit code to GENERAL_ERROR', () => { + const err = new RetryableError('test'); + expect(err.code).toBe(ExitCode.GENERAL_ERROR); + }); + + it('passes message through', () => { + const err = new RetryableError('something went wrong'); + expect(err.message).toBe('something went wrong'); + }); + + it('accepts an optional cause', () => { + const cause = new Error('original'); + const err = new RetryableError('wrapped', cause); + expect(err.cause).toBe(cause); + }); + + it('accepts an optional retryAfter (ms)', () => { + const err = new RetryableError('rate limited', undefined, 5000); + expect(err.retryAfter).toBe(5000); + }); + + it('defaults retryAfter to undefined', () => { + const err = new RetryableError('test'); + expect(err.retryAfter).toBeUndefined(); + }); + + it('is identified by isCCSError', () => { + const err = new RetryableError('test'); + expect(isCCSError(err)).toBe(true); + }); + + it('is identified as recoverable by isRecoverableError', () => { + const err = new RetryableError('test'); + expect(isRecoverableError(err)).toBe(true); + }); + + it('has a proper stack trace', () => { + const err = new RetryableError('test'); + expect(err.stack).toBeDefined(); + expect(err.stack).toContain('RetryableError'); + }); +}); diff --git a/src/errors/error-types.ts b/src/errors/error-types.ts index 08424dfe..c0b8a9b2 100644 --- a/src/errors/error-types.ts +++ b/src/errors/error-types.ts @@ -169,6 +169,21 @@ export class ValidationError extends CCSError { } } +/** + * Retryable/transient error + * Signals that the operation may succeed on retry (e.g. rate limits, timeouts) + */ +export class RetryableError extends CCSError { + constructor( + message: string, + public readonly cause?: Error, + public readonly retryAfter?: number // ms until next attempt + ) { + super(message, ExitCode.GENERAL_ERROR, true); + this.name = 'RetryableError'; + } +} + /** * Type guard to check if an error is a CCSError */ diff --git a/src/errors/index.ts b/src/errors/index.ts index 40d2cd4b..45e89b7d 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -37,6 +37,7 @@ export { ProxyError, MigrationError, UserAbortError, + RetryableError, isCCSError, isRecoverableError, } from './error-types'; diff --git a/src/utils/__tests__/retry-strategy.test.ts b/src/utils/__tests__/retry-strategy.test.ts new file mode 100644 index 00000000..e22693ef --- /dev/null +++ b/src/utils/__tests__/retry-strategy.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect, mock, spyOn } from 'bun:test'; +import { withRetry, type RetryOptions } from '../retry-strategy'; +import { RetryableError } from '../../errors/error-types'; + +describe('withRetry', () => { + it('returns the result on first success', async () => { + const fn = mock(() => Promise.resolve(42)); + const result = await withRetry(fn, { maxRetries: 3, baseDelayMs: 10 }); + expect(result).toBe(42); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('retries on RetryableError and succeeds', async () => { + let attempt = 0; + const fn = mock(() => { + attempt++; + if (attempt < 3) { + return Promise.reject(new RetryableError('transient failure')); + } + return Promise.resolve('ok'); + }); + + const result = await withRetry(fn, { maxRetries: 5, baseDelayMs: 1 }); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('throws after max retries exhausted', async () => { + const fn = mock(() => Promise.reject(new RetryableError('always fails'))); + await expect(withRetry(fn, { maxRetries: 2, baseDelayMs: 1 })).rejects.toThrow('always fails'); + // 1 initial + 2 retries = 3 total calls + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('does not retry non-retryable errors', async () => { + const fn = mock(() => Promise.reject(new Error('fatal'))); + await expect(withRetry(fn, { maxRetries: 3, baseDelayMs: 1 })).rejects.toThrow('fatal'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('does not retry errors with recoverable=false', async () => { + const fn = mock(() => Promise.reject(new Error('non-retryable'))); + await expect(withRetry(fn, { maxRetries: 3, baseDelayMs: 1 })).rejects.toThrow('non-retryable'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('uses custom retryableCheck when provided', async () => { + let attempt = 0; + const fn = mock(() => { + attempt++; + if (attempt < 2) { + return Promise.reject(new Error('custom-retry')); + } + return Promise.resolve('recovered'); + }); + + const customCheck = (error: unknown) => + error instanceof Error && error.message === 'custom-retry'; + + const result = await withRetry(fn, { + maxRetries: 5, + baseDelayMs: 1, + retryableCheck: customCheck, + }); + expect(result).toBe('recovered'); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('calls onRetry callback on each retry attempt', async () => { + const onRetry = mock(() => {}); + const fn = mock(() => Promise.reject(new RetryableError('fail'))); + + await expect(withRetry(fn, { maxRetries: 3, baseDelayMs: 1, onRetry })).rejects.toThrow('fail'); + + // 1 initial + 3 retries = 3 onRetry calls (not called for initial) + expect(onRetry).toHaveBeenCalledTimes(3); + // First retry call + expect(onRetry.mock.calls[0][1]).toBe(1); + }); + + it('respects maxDelayMs cap', async () => { + const sleepSpy = spyOn(globalThis, 'setTimeout'); + let attempt = 0; + const fn = mock(() => { + attempt++; + if (attempt <= 2) { + return Promise.reject(new RetryableError('fail')); + } + return Promise.resolve('ok'); + }); + + await withRetry(fn, { + maxRetries: 5, + baseDelayMs: 1000, + maxDelayMs: 200, + }); + + // Verify setTimeout was called with delay <= maxDelayMs (200ms) + jitter buffer + // Jitter adds 0-20% of delay, so max possible is 240ms + for (const call of sleepSpy.mock.calls) { + const delay = call[1] as number; + expect(delay).toBeLessThanOrEqual(250); // allow small jitter overhead + } + sleepSpy.mockRestore(); + }); + + it('uses default backoffMultiplier when not specified', async () => { + let attempt = 0; + const fn = mock(() => { + attempt++; + if (attempt < 2) { + return Promise.reject(new RetryableError('fail')); + } + return Promise.resolve('ok'); + }); + + // Should not throw - defaults to multiplier of 2 + await expect(withRetry(fn, { maxRetries: 3, baseDelayMs: 1 })).resolves.toBe('ok'); + }); + + it('applies exponential backoff with custom multiplier', async () => { + const sleepSpy = spyOn(globalThis, 'setTimeout'); + let attempt = 0; + const fn = mock(() => { + attempt++; + if (attempt <= 3) { + return Promise.reject(new RetryableError('fail')); + } + return Promise.resolve('ok'); + }); + + await withRetry(fn, { + maxRetries: 5, + baseDelayMs: 10, + backoffMultiplier: 3, + }); + + // Delays should grow: ~10, ~30, ~90 (with jitter) + const delays = sleepSpy.mock.calls.map((call) => call[1] as number); + // First delay should be close to base * multiplier^0 = 10 + expect(delays[0]).toBeGreaterThan(5); + expect(delays[0]).toBeLessThan(25); // 10 + jitter + // Second delay should be close to base * multiplier^1 = 30 + expect(delays[1]).toBeGreaterThan(20); + expect(delays[1]).toBeLessThan(50); // 30 + jitter + + sleepSpy.mockRestore(); + }); + + it('passes through errors that are not Error instances', async () => { + const fn = mock(() => Promise.reject('string error')); + await expect(withRetry(fn, { maxRetries: 3, baseDelayMs: 1 })).rejects.toBe('string error'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('works with maxRetries of 0 (no retries)', async () => { + const fn = mock(() => Promise.reject(new RetryableError('fail'))); + await expect(withRetry(fn, { maxRetries: 0, baseDelayMs: 1 })).rejects.toThrow('fail'); + expect(fn).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/utils/retry-strategy.ts b/src/utils/retry-strategy.ts new file mode 100644 index 00000000..988f9216 --- /dev/null +++ b/src/utils/retry-strategy.ts @@ -0,0 +1,117 @@ +/** + * Retry Strategy Utility + * + * Reusable exponential-backoff retry wrapper extracted from + * scattered retry logic in glmt-proxy and binary/downloader. + * + * Usage: + * const data = await withRetry(() => fetch(url), { maxRetries: 3, baseDelayMs: 100 }); + */ + +import { RetryableError, isRecoverableError } from '../errors/error-types'; + +/** Configuration options for retry behavior */ +export interface RetryOptions { + /** Maximum number of retry attempts (default: 3) */ + maxRetries: number; + /** Base delay in ms for the first retry (default: 1000) */ + baseDelayMs: number; + /** Upper bound for the computed delay (default: 30000) */ + maxDelayMs?: number; + /** Multiplier applied per attempt (default: 2) */ + backoffMultiplier?: number; + /** Override the default retryability check */ + retryableCheck?: (error: unknown) => boolean; + /** Callback fired before each retry (not fired on initial call) */ + onRetry?: (error: Error, attempt: number) => void; +} + +const DEFAULT_MAX_DELAY_MS = 30_000; +const DEFAULT_MULTIPLIER = 2; +const JITTER_RATIO = 0.2; // 20% of delay as random jitter + +/** + * Check whether an unknown thrown value is retryable. + * Uses CCSError.recoverable flag and RetryableError instance check. + */ +function defaultRetryableCheck(error: unknown): boolean { + if (error instanceof RetryableError) { + return true; + } + if (isRecoverableError(error)) { + return true; + } + return false; +} + +/** + * Compute backoff delay: base * multiplier^attempt + jitter, capped at maxDelay. + */ +function computeDelay( + attempt: number, + baseDelayMs: number, + maxDelayMs: number, + multiplier: number +): number { + const exponentialDelay = Math.min(baseDelayMs * Math.pow(multiplier, attempt), maxDelayMs); + const jitter = exponentialDelay * JITTER_RATIO * Math.random(); + return exponentialDelay + jitter; +} + +/** + * Sleep for the specified number of milliseconds. + */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Execute `fn` with automatic retries on retryable failures. + * + * Retryability defaults to checking for `RetryableError` instances + * and CCSError with `recoverable === true`. Override with `retryableCheck`. + * + * @param fn - The async function to execute + * @param options - Retry configuration + * @returns The resolved value from `fn` + * @throws The last error encountered after exhausting retries + */ +export async function withRetry(fn: () => Promise, options: RetryOptions): Promise { + const { + maxRetries, + baseDelayMs, + maxDelayMs = DEFAULT_MAX_DELAY_MS, + backoffMultiplier = DEFAULT_MULTIPLIER, + retryableCheck = defaultRetryableCheck, + onRetry, + } = options; + + const isRetryable = retryableCheck ?? defaultRetryableCheck; + let lastError: unknown; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + + // No more retries left + if (attempt >= maxRetries) { + break; + } + + // Check retryability + if (!isRetryable(error)) { + break; + } + + const err = error instanceof Error ? error : new Error(String(error)); + onRetry?.(err, attempt + 1); + + const delay = computeDelay(attempt, baseDelayMs, maxDelayMs, backoffMultiplier); + await sleep(delay); + } + } + + throw lastError; +}