tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-02 06:22:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(anthropic): route Claude Opus 4.8 through adaptive thinking (#29702)

Browse Source 
* fix(anthropic): route Claude Opus 4.8 through adaptive thinking

Opus 4.8 uses the same adaptive thinking contract as 4.6/4.7
(thinking.type=adaptive plus output_config.effort), but
_is_adaptive_thinking_model only recognized 4.6/4.7 by name and otherwise
leaned on the supports_adaptive_thinking cost-map flag. The Bedrock,
Vertex, and Azure 4.8 entries don't carry that flag, so a
bedrock/us.anthropic.claude-opus-4-8 request fell back to the legacy
thinking.type=enabled shape and Bedrock rejected it with "thinking.type.enabled
is not supported for this model".

Add _is_claude_4_8_model and wire it in next to the existing 4.6/4.7
matchers in the adaptive-thinking detection, the effort=max gate, and the
supported-params check, so every provider path treats 4.8 as adaptive
regardless of whether its cost-map entry advertises the flag.

* refactor(anthropic): drive Opus 4.8 adaptive thinking from the cost map

Replace the _is_claude_4_8_model name matcher with cost-map data. Add
supports_adaptive_thinking to every Opus 4.8 provider variant (Bedrock
regional/global, Vertex, Azure) in both the root and bundled cost maps, and
move the prefix-resolving capability lookup (_supports_model_capability) down
to AnthropicModelInfo so _is_adaptive_thinking_model reads the flag through the
bedrock/invoke/, bedrock/, and vertex_ai/ prefixes. The 4.6/4.7 name checks
stay as a fallback since their provider entries don't carry the flag yet.

A pure data fix is not enough on its own: _supports_factory doesn't strip the
us.anthropic./invoke/ prefixes, so bedrock/invoke/us.anthropic.claude-opus-4-8
would still miss the flag without the resolver change.

Add a cost-map guardrail test asserting every claude-opus-4-8 variant carries
the flag, so a future variant added without it fails CI instead of silently
sending the legacy thinking.type=enabled shape that the provider rejects.
This commit is contained in:
Mateo Wang
2026-06-05 16:19:01 +05:30
committed by GitHub
parent 8259d6cd85
commit 1c741b91c0
7 changed files with 208 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/anthropic/chat/transformation.py
-45
View File
@@ -81,7 +81,6 @@ from litellm.types.utils import (
from litellm.utils import (
ModelResponse,
Usage,
_supports_factory,
add_dummy_tool,
any_assistant_message_has_thinking_blocks,
get_max_tokens,
@@ -337,50 +336,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7")
)
@staticmethod
def _supports_model_capability(model: str, key: str) -> bool:
"""Check a boolean capability ``key`` in the model map.
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
"""
try:
if _supports_factory(
model=model,
custom_llm_provider="anthropic",
key=key,
):
return True
except Exception:
pass
candidates = [model]
for prefix in (
"bedrock/converse/",
"bedrock/invoke/",
"bedrock/",
"vertex_ai/",
):
if model.startswith(prefix):
candidates.append(model[len(prefix) :])
try:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
base = BedrockModelInfo.get_base_model(model)
if base:
candidates.append(base)
candidates.append(f"bedrock/{base}")
except Exception:
pass
try:
for cand in candidates:
if cand in litellm.model_cost and (
litellm.model_cost[cand].get(key) is True
):
return True
except Exception:
pass
return False
@staticmethod
def _supports_effort_level(model: str, level: str) -> bool:
"""Check ``supports_{level}_reasoning_effort`` in the model map."""
litellm/llms/anthropic/common_utils.py
+48 -4
View File
@@ -272,19 +272,63 @@ class AnthropicModelInfo(BaseLLMModelInfo):
)
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:
"""Claude 4.6+ models use adaptive thinking with ``output_config.effort``."""
def _supports_model_capability(model: str, key: str) -> bool:
"""Check a boolean capability ``key`` in the model map.
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
"""
from litellm.utils import _supports_factory
try:
if _supports_factory(
model=model,
custom_llm_provider=None,
key="supports_adaptive_thinking",
custom_llm_provider="anthropic",
key=key,
):
return True
except Exception:
pass
candidates = [model]
for prefix in (
"bedrock/converse/",
"bedrock/invoke/",
"bedrock/",
"vertex_ai/",
):
if model.startswith(prefix):
candidates.append(model[len(prefix) :])
try:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
base = BedrockModelInfo.get_base_model(model)
if base:
candidates.append(base)
candidates.append(f"bedrock/{base}")
except Exception:
pass
try:
for cand in candidates:
if cand in litellm.model_cost and (
litellm.model_cost[cand].get(key) is True
):
return True
except Exception:
pass
return False
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:
"""Claude 4.6+ models use adaptive thinking with ``output_config.effort``.
Driven by the ``supports_adaptive_thinking`` flag in the model map; the
4.6/4.7 name checks remain only as a fallback for provider-routed ids
whose map entries predate the flag.
"""
if AnthropicModelInfo._supports_model_capability(
model, "supports_adaptive_thinking"
):
return True
return AnthropicModelInfo._is_claude_4_6_model(
model
) or AnthropicModelInfo._is_claude_4_7_model(model)
litellm/model_prices_and_context_window_backup.json
+8
View File
@@ -1319,6 +1319,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1350,6 +1351,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1381,6 +1383,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1412,6 +1415,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1443,6 +1447,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -2194,6 +2199,7 @@
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -33927,6 +33933,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -33955,6 +33962,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
model_prices_and_context_window.json
+8
View File
@@ -1319,6 +1319,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1350,6 +1351,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1381,6 +1383,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1412,6 +1415,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -1443,6 +1447,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -2194,6 +2199,7 @@
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -33962,6 +33968,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -33990,6 +33997,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py
+51
View File
@@ -2,6 +2,7 @@
import pytest
import litellm
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@@ -11,6 +12,22 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran
)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force the bundled backup cost map so Opus 4.8 adaptive detection (driven
by the ``supports_adaptive_thinking`` flag) doesn't depend on the
network-fetched ``main`` copy, which lacks the flag until this branch merges."""
original = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.get_model_info.cache_clear()
try:
yield
finally:
litellm.model_cost = original
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize(
"reasoning_effort,expected_effort",
[
@@ -298,6 +315,40 @@ def test_legacy_thinking_high_budget_keeps_xhigh_when_supported():
assert result.get("output_config") == {"effort": "xhigh"}
@pytest.mark.parametrize(
"model",
[
"claude-opus-4-8",
"bedrock/us.anthropic.claude-opus-4-8",
"bedrock/invoke/us.anthropic.claude-opus-4-8",
],
)
def test_legacy_thinking_translates_to_adaptive_for_opus_48(
model, local_model_cost_map
):
"""Regression for issue #29188: Opus 4.8 requires adaptive thinking, but the
legacy ``thinking.type='enabled'`` shape was passed through unchanged for
Bedrock 4.8 (its cost-map entry lacked ``supports_adaptive_thinking`` and the
lookup didn't strip the provider prefix), so Bedrock rejected the request. The
reporter's reproducer used ``budget_tokens=24000``, the ``xhigh`` bucket."""
config = AnthropicMessagesConfig()
optional_params = {
"max_tokens": 100,
"thinking": {"type": "enabled", "budget_tokens": 24000},
}
result = config.transform_anthropic_messages_request(
model=model,
messages=[{"role": "user", "content": "ping"}],
anthropic_messages_optional_request_params=optional_params,
litellm_params={},
headers={},
)
assert result.get("thinking") == {"type": "adaptive"}
assert result.get("output_config") == {"effort": "xhigh"}
@pytest.mark.parametrize(
"budget_tokens,expected_effort",
[
tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py
+72
View File
@@ -14,6 +14,8 @@ import os
import sys
from unittest.mock import patch
import pytest
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
)
@@ -1378,3 +1380,73 @@ class TestAnthropicThinkingSignatureSelfHeal:
config.transform_anthropic_messages_request_on_http_error(err, data)
assert "thinking" not in data
assert data["messages"] == []
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force the bundled backup cost map so detection doesn't depend on the
network-fetched ``main`` copy (which lacks this branch's flags until merge)."""
import litellm
original = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.get_model_info.cache_clear()
try:
yield
finally:
litellm.model_cost = original
litellm.get_model_info.cache_clear()
class TestClaudeOpus48AdaptiveThinking:
"""Opus 4.8 requires adaptive thinking (``thinking.type='adaptive'`` +
``output_config.effort``). Detection is driven by the
``supports_adaptive_thinking`` cost-map flag, resolved through provider
prefixes. Before the fix the Bedrock entries lacked the flag and the lookup
didn't strip the ``us.anthropic.``/``invoke/`` prefixes, so a
``bedrock/us.anthropic.claude-opus-4-8`` call sent the legacy
``thinking.type='enabled'`` shape and Bedrock rejected it (issue #29188)."""
@pytest.mark.parametrize(
"model",
[
"claude-opus-4-8",
"anthropic/claude-opus-4-8",
"anthropic.claude-opus-4-8",
"bedrock/us.anthropic.claude-opus-4-8",
"bedrock/invoke/us.anthropic.claude-opus-4-8",
"bedrock/eu.anthropic.claude-opus-4-8",
"vertex_ai/claude-opus-4-8",
"azure_ai/claude-opus-4-8",
],
)
def test_adaptive_thinking_detected_for_opus_4_8(self, local_model_cost_map, model):
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True
def test_resolver_reads_flag_through_bedrock_invoke_prefix(
self, local_model_cost_map
):
"""The resolver fix: ``bedrock/invoke/...`` resolves to the flagged
Bedrock entry. Pure ``_supports_factory`` without prefix-stripping
returns False here, which is why the data-only fix alone was not enough."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert (
AnthropicModelInfo._supports_model_capability(
"bedrock/invoke/us.anthropic.claude-opus-4-8",
"supports_adaptive_thinking",
)
is True
)
@pytest.mark.parametrize(
"model",
["claude-opus-4-5", "claude-3-7-sonnet", "claude-3-5-haiku-20241022"],
)
def test_non_adaptive_models_not_detected(self, local_model_cost_map, model):
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model) is False
tests/test_litellm/test_claude_opus_4_8_config.py
+21
View File
@@ -182,3 +182,24 @@ def test_opus_4_8_provider_resolves_via_model_info(local_model_cost_map):
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_opus_4_8_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Opus 4.8 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s (issue #29188, which the Bedrock/Vertex/Azure variants hit
because only the bare ``claude-opus-4-8`` entry carried the flag). This guards
against a future variant being added without it."""
variants = [k for k in cost_map if "claude-opus-4-8" in k]
assert variants, "no claude-opus-4-8 entries found in cost map"
missing = [
k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True
]
assert not missing, f"missing supports_adaptive_thinking: {missing}"