/** * Composite Variant Fallback Tests * * Tests fallback detection and application for composite variants */ import { describe, it, expect } from 'bun:test'; import { isProviderError, detectFailedTier, PROVIDER_ERROR_PATTERNS, } from '../../../src/cliproxy/executor/retry-handler'; import { applyFallback } from '../../../src/cliproxy/executor/env-resolver'; import { CompositeTierConfig } from '../../../src/config/unified-config-types'; // ======================================== // isProviderError // ======================================== describe('isProviderError', () => { it('should return false for exit code 0 (success)', () => { const result = isProviderError(0, 'Error: 500 Internal Server Error'); expect(result).toBe(false); }); it('should detect 4xx error codes', () => { const stderr = 'Error: 401 Unauthorized'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should detect 5xx error codes', () => { const stderr = 'Error: 503 Service Unavailable'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should detect overloaded errors', () => { const stderr = 'Provider is currently overloaded. Please try again later.'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should detect quota exceeded errors', () => { const stderr = 'quota has been exceeded for this account'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should detect connection refused errors', () => { const stderr = 'ECONNREFUSED 127.0.0.1:8317'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should detect rate limit errors', () => { const stderr = 'Rate limit exceeded. Please slow down.'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should return false for normal user exit', () => { const stderr = 'User cancelled the operation'; const result = isProviderError(1, stderr); expect(result).toBe(false); }); it('should return false for empty stderr with non-zero exit', () => { const result = isProviderError(1, ''); expect(result).toBe(false); }); it('should be case insensitive for error patterns', () => { const stderr = 'ERROR: OVERLOADED'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); it('should handle multiple error patterns in same stderr', () => { const stderr = 'Error: 429 rate limit exceeded'; const result = isProviderError(1, stderr); expect(result).toBe(true); }); }); // ======================================== // detectFailedTier // ======================================== describe('detectFailedTier', () => { const tiers: { opus: CompositeTierConfig; sonnet: CompositeTierConfig; haiku: CompositeTierConfig; } = { opus: { provider: 'agy', model: 'claude-opus-4-6-thinking' }, sonnet: { provider: 'gemini', model: 'gemini-3-pro-preview' }, haiku: { provider: 'codex', model: 'gpt-4o-mini' }, }; it('should detect opus tier from model name in stderr', () => { const stderr = 'Error calling model claude-opus-4-6-thinking: 503 overloaded'; const result = detectFailedTier(stderr, tiers); expect(result).toBe('opus'); }); it('should detect sonnet tier from model name in stderr', () => { const stderr = 'Failed to connect: gemini-3-pro-preview returned 500'; const result = detectFailedTier(stderr, tiers); expect(result).toBe('sonnet'); }); it('should detect haiku tier from model name in stderr', () => { const stderr = 'gpt-4o-mini: rate limit exceeded'; const result = detectFailedTier(stderr, tiers); expect(result).toBe('haiku'); }); it('should return null when no tier model found in stderr', () => { const stderr = 'Generic error with no model mention'; const result = detectFailedTier(stderr, tiers); expect(result).toBe(null); }); it('should return null for empty stderr', () => { const result = detectFailedTier('', tiers); expect(result).toBe(null); }); it('should match first tier when multiple models mentioned', () => { const stderr = 'Tried claude-opus-4-6-thinking, then gemini-3-pro-preview, both failed'; const result = detectFailedTier(stderr, tiers); expect(result).toBe('opus'); // First match }); it('should handle model names with suffixes (thinking budgets)', () => { const tiersWithBudget: typeof tiers = { opus: { provider: 'agy', model: 'claude-opus-4-6-thinking(high)' }, sonnet: { provider: 'gemini', model: 'gemini-3-pro-preview(medium)' }, haiku: { provider: 'codex', model: 'gpt-4o-mini' }, }; // Stderr might not include suffix const stderr = 'Error with claude-opus-4-6-thinking: timeout'; const result = detectFailedTier(stderr, tiersWithBudget); // Should still match because model name is substring expect(result).toBe('opus'); }); it('should strip complex thinking suffix with comma-separated params', () => { const tiersWithComplexBudget: typeof tiers = { opus: { provider: 'agy', model: 'claude-opus-4-6-thinking(32768,extended)' }, sonnet: { provider: 'agy', model: 'claude-sonnet-4-5-thinking(high)' }, haiku: { provider: 'agy', model: 'claude-3-5-haiku' }, }; // Stderr contains base model name without suffix const stderr = 'Error: claude-opus-4-6-thinking overloaded'; const result = detectFailedTier(stderr, tiersWithComplexBudget); // Should match because regex strips (32768,extended) to get base name expect(result).toBe('opus'); }); }); // ======================================== // applyFallback // ======================================== describe('applyFallback', () => { it('should update opus tier env var', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const result = applyFallback(env, 'opus', { provider: 'gemini', model: 'gemini-3-pro-preview', }); expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('gemini-3-pro-preview'); // Should not modify other tiers expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking'); expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4-5-20251001'); }); it('should update sonnet tier env var', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const result = applyFallback(env, 'sonnet', { provider: 'codex', model: 'gpt-4o', }); expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('gpt-4o'); expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking'); expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4-5-20251001'); }); it('should update haiku tier env var', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-haiku-4-5-20251001', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const result = applyFallback(env, 'haiku', { provider: 'gemini', model: 'gemini-3-flash-preview', }); expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('gemini-3-flash-preview'); expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking'); expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking'); }); it('should update ANTHROPIC_MODEL when failed tier is default tier', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const result = applyFallback(env, 'opus', { provider: 'gemini', model: 'gemini-3-pro-preview', }); // Both tier model and default model should be updated expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('gemini-3-pro-preview'); expect(result.ANTHROPIC_MODEL).toBe('gemini-3-pro-preview'); }); it('should NOT update ANTHROPIC_MODEL when failed tier is not default tier', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-sonnet-4-5-thinking', // Sonnet is default ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const result = applyFallback(env, 'opus', { provider: 'gemini', model: 'gemini-3-pro-preview', }); // Opus tier should be updated but ANTHROPIC_MODEL should remain sonnet expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('gemini-3-pro-preview'); expect(result.ANTHROPIC_MODEL).toBe('claude-sonnet-4-5-thinking'); }); it('should preserve other env vars', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', CUSTOM_VAR: 'custom_value', ANOTHER_VAR: '12345', }; const result = applyFallback(env, 'opus', { provider: 'gemini', model: 'gemini-3-pro-preview', }); expect(result.CUSTOM_VAR).toBe('custom_value'); expect(result.ANOTHER_VAR).toBe('12345'); expect(result.ANTHROPIC_BASE_URL).toBe('http://127.0.0.1:8318'); }); it('should not mutate original env object', () => { const env = { ANTHROPIC_BASE_URL: 'http://127.0.0.1:8318', ANTHROPIC_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-opus-4-6-thinking', ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-sonnet-4-5-thinking', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-4-5-20251001', }; const original = { ...env }; applyFallback(env, 'opus', { provider: 'gemini', model: 'gemini-3-pro-preview', }); // Original env should be unchanged expect(env.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe(original.ANTHROPIC_DEFAULT_OPUS_MODEL); }); }); // ======================================== // PROVIDER_ERROR_PATTERNS // ======================================== describe('PROVIDER_ERROR_PATTERNS', () => { it('should export error pattern regexes', () => { expect(PROVIDER_ERROR_PATTERNS).toBeDefined(); expect(Array.isArray(PROVIDER_ERROR_PATTERNS)).toBe(true); expect(PROVIDER_ERROR_PATTERNS.length).toBeGreaterThan(0); }); it('should match 4xx errors', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('Error: 401')); expect(pattern).toBeDefined(); }); it('should match 5xx errors', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('Error: 503')); expect(pattern).toBeDefined(); }); it('should match overloaded keyword', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('overloaded')); expect(pattern).toBeDefined(); }); it('should match quota exceeded', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('quota exceeded')); expect(pattern).toBeDefined(); }); it('should match ECONNREFUSED', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('ECONNREFUSED')); expect(pattern).toBeDefined(); }); it('should match rate limit', () => { const pattern = PROVIDER_ERROR_PATTERNS.find((p) => p.test('rate limit')); expect(pattern).toBeDefined(); }); });