From 88d1dc1ddb6dc2807edf89045b808cdb77c0bf3b Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 10 Mar 2026 17:34:45 +0700 Subject: [PATCH] fix(cliproxy): validate tier thinking against model caps --- src/cliproxy/config/thinking-config.ts | 10 +++++ src/cliproxy/thinking-validator.ts | 26 ++++++++++++ .../unit/cliproxy/composite-thinking.test.ts | 42 +++++++++---------- .../unit/cliproxy/thinking-validator.test.ts | 7 ++++ 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/cliproxy/config/thinking-config.ts b/src/cliproxy/config/thinking-config.ts index 5afe0b3b..1162b561 100644 --- a/src/cliproxy/config/thinking-config.ts +++ b/src/cliproxy/config/thinking-config.ts @@ -303,6 +303,16 @@ export function applyThinkingConfig( continue; } + // Validate/clamp tier thinking against this specific tier model capabilities. + const tierValidation = validateThinking(tierProvider, normalizedTierModel, tierThinkingValue); + if (tierValidation.warning && shouldShowWarnings(thinkingConfig)) { + console.warn(warn(tierValidation.warning)); + } + tierThinkingValue = tierValidation.value; + if (isThinkingOffValue(tierThinkingValue)) { + continue; + } + result[tierVar] = applyThinkingSuffixForProvider(model, tierThinkingValue, tierProvider); } } diff --git a/src/cliproxy/thinking-validator.ts b/src/cliproxy/thinking-validator.ts index 8b14ed95..e749ecb7 100644 --- a/src/cliproxy/thinking-validator.ts +++ b/src/cliproxy/thinking-validator.ts @@ -382,6 +382,32 @@ function validateLevelThinking( ); } + // If the input is a standard named level but this model only supports a subset, + // choose the highest supported level that does not exceed the requested intensity. + const requestedRank = THINKING_LEVEL_RANK[normalizedLevel]; + if (requestedRank && validLevels.length > 0) { + const rankedValidLevels = validLevels + .filter((level) => THINKING_LEVEL_RANK[level] !== undefined) + .sort((a, b) => (THINKING_LEVEL_RANK[a] ?? 0) - (THINKING_LEVEL_RANK[b] ?? 0)); + + if (rankedValidLevels.length > 0) { + let mappedLevel = rankedValidLevels[0]; + for (const level of rankedValidLevels) { + const levelRank = THINKING_LEVEL_RANK[level] ?? 0; + if (levelRank <= requestedRank) { + mappedLevel = level; + continue; + } + break; + } + + return applyMaxCap( + mappedLevel, + `Level "${value}" mapped to "${mappedLevel}" for ${modelId} (available: ${validLevels.join(', ')}).` + ); + } + } + // Try to map from standard level names to model's levels const standardToModelLevel: Record = {}; const levelOrder = ['minimal', 'low', 'medium', 'high', 'xhigh']; diff --git a/tests/unit/cliproxy/composite-thinking.test.ts b/tests/unit/cliproxy/composite-thinking.test.ts index fe1ad59c..6fb3758b 100644 --- a/tests/unit/cliproxy/composite-thinking.test.ts +++ b/tests/unit/cliproxy/composite-thinking.test.ts @@ -192,9 +192,9 @@ describe('applyThinkingConfig - composite variant integration', () => { compositeTierThinking ); - // Tier models use raw compositeTierThinking value (no validation in tier loop) - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(xhigh)'); - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(medium)'); + // Budget-based tier models are normalized to provider-specific budget values. + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(32768)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(8192)'); // Haiku doesn't support thinking per model-catalog — no suffix expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4-5-20251001'); // ANTHROPIC_MODEL is validated: 'medium' → 8192 for budget-type models @@ -221,8 +221,8 @@ describe('applyThinkingConfig - composite variant integration', () => { compositeTierThinking ); - // Tier models use raw value (no validation in tier loop) - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(xhigh)'); + // Budget-based tier models are normalized to provider-specific budget values. + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(32768)'); // Sonnet gets defaults from global config (mode=auto) expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toContain('claude-sonnet-4-5-thinking'); // Haiku doesn't support thinking — stays unchanged @@ -250,8 +250,8 @@ describe('applyThinkingConfig - composite variant integration', () => { compositeTierThinking ); - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(high)'); - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(medium)'); + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(24576)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(8192)'); // Haiku should NOT have suffix because thinking is not supported for haiku expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4-5-20251001'); }); @@ -277,9 +277,9 @@ describe('applyThinkingConfig - composite variant integration', () => { compositeTierThinking ); - // All thinking-capable tiers should use CLI override - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(minimal)'); - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(minimal)'); + // All thinking-capable tiers should use the validated CLI override. + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(1024)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(1024)'); // Haiku doesn't support thinking — stays unchanged regardless of CLI override expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4-5-20251001'); }); @@ -349,7 +349,7 @@ describe('applyThinkingConfig - composite variant integration', () => { } ); - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(high)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(24576)'); }); it('applies tier thinking even when default model does not support thinking', () => { @@ -375,7 +375,7 @@ describe('applyThinkingConfig - composite variant integration', () => { // Default model provider/model do not support thinking. expect(result.ANTHROPIC_MODEL).toBe('gpt-4o-mini'); // Supported mixed-provider tier must still receive its configured thinking value. - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(high)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(24576)'); }); it('handles dotted agy Claude IDs for thinking capability lookup', () => { @@ -389,10 +389,10 @@ describe('applyThinkingConfig - composite variant integration', () => { const result = applyThinkingConfig(envVars, 'agy' as CLIProxyProvider, 'high'); // Capability lookup should succeed for dotted agy IDs after normalization. - // Main model value is validated for budget models; tier values keep raw override. + // Tier values are normalized through budget validation too. expect(result.ANTHROPIC_MODEL).toBe('claude-opus-4.5-thinking(24576)'); - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4.5-thinking(high)'); - expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4.5-thinking(high)'); + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4.5-thinking(24576)'); + expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4.5-thinking(24576)'); // Haiku does not support thinking in agy catalog. expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('claude-haiku-4.5'); }); @@ -418,7 +418,7 @@ describe('applyThinkingConfig - composite variant integration', () => { compositeTierThinking ); - // Tier models use raw string values (no validation in tier loop) + // Tier models keep numeric values after validation when the provider supports them. expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(32768)'); expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(8192)'); // Haiku doesn't support thinking — stays unchanged @@ -506,8 +506,8 @@ describe('applyThinkingConfig - composite variant integration', () => { // ANTHROPIC_MODEL is validated: xhigh → 32768 for budget-type models expect(result.ANTHROPIC_MODEL).toBe('claude-opus-4-6-thinking(32768)'); - // Tier model uses raw value (no validation in tier loop) - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(xhigh)'); + // Tier model keeps the validated budget value. + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(32768)'); }); it('should preserve models that already have thinking suffix', () => { @@ -534,8 +534,8 @@ describe('applyThinkingConfig - composite variant integration', () => { // Existing suffix on main model should be preserved. expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('claude-sonnet-4-5-thinking(high)'); expect(result.ANTHROPIC_MODEL).toBe('claude-sonnet-4-5-thinking(high)'); - // Other supported tiers still receive configured thinking. - expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(xhigh)'); + // Other supported tiers still receive configured thinking after validation. + expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('claude-opus-4-6-thinking(32768)'); }); it('should use codex effort suffix style when provider is codex', () => { @@ -556,7 +556,7 @@ describe('applyThinkingConfig - composite variant integration', () => { expect(result.ANTHROPIC_MODEL).toBe('gpt-5.3-codex-xhigh'); expect(result.ANTHROPIC_DEFAULT_OPUS_MODEL).toBe('gpt-5.3-codex-xhigh'); expect(result.ANTHROPIC_DEFAULT_SONNET_MODEL).toBe('gpt-5.3-codex-xhigh'); - expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('gpt-5-mini-xhigh'); + expect(result.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('gpt-5-mini-high'); }); it('should normalize legacy codex parenthesized tier suffix to effort suffix format', () => { diff --git a/tests/unit/cliproxy/thinking-validator.test.ts b/tests/unit/cliproxy/thinking-validator.test.ts index 0be4ac61..d48679be 100644 --- a/tests/unit/cliproxy/thinking-validator.test.ts +++ b/tests/unit/cliproxy/thinking-validator.test.ts @@ -102,6 +102,13 @@ describe('Thinking Validator', () => { expect(typeof result.value).toBe('string'); // Should be a level name expect(result.warning).toContain('Mapped'); }); + + it('caps unsupported higher standard levels to the highest supported model level', () => { + const result = validateThinking('codex', 'gpt-5-mini', 'xhigh'); + expect(result.valid).toBe(true); + expect(result.value).toBe('high'); + expect(result.warning).toContain('mapped to "high"'); + }); }); describe('Budget-type models (like Claude via agy)', () => {