test(ci): add reasoning_effort grid v4 e2e regression suite

Encode the 231-cell QA sweep (21 provider x model combos x 11 effort
values) from #27039 / #27074 as an automated CircleCI-gated regression
suite. Each cell hits the real provider endpoint, captures the outgoing
wire body via a pre-call CustomLogger, and asserts:

- thinking.type, output_config.effort, thinking.budget_tokens, max_tokens
  in the captured request body (regression signal for silent drops/strips
  in any provider transformation)
- HTTP status (200 vs BadRequestError -> 400) returned by litellm
  (regression signal for clean-error vs leaked-500 mappings)

The matrix is encoded as a small rule set keyed by (model_mode, effort)
plus per-model xhigh/max capability overrides, then expanded across the
five chat-completion routes (Anthropic direct, Azure AI Foundry, Vertex
AI, Bedrock Converse, Bedrock Invoke /chat) and the Bedrock Invoke
/v1/messages route. Cells skip at runtime when the route's provider env
vars are absent, so PR builds without credentials no-op gracefully.

Wired into CircleCI as the reasoning_effort_grid_v4_e2e job behind the
existing main / litellm_* branch filter.

.circleci/config.yml

@@ -576,6 +576,48 @@ jobs:
           no_output_timeout: 15m
 
       # Store test results
+      - store_test_results:
+          path: test-results
+  reasoning_effort_grid_v4_e2e:
+    docker:
+      - *python312_image
+    working_directory: ~/project
+    resource_class: large
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - restore_cache:
+          keys:
+            - v1-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+      - save_cache:
+          paths:
+            - ~/.cache/uv
+          key: v1-uv-cache-{{ checksum "uv.lock" }}
+      # Grid v4 exercises reasoning_effort mapping against real Anthropic,
+      # Azure AI Foundry, Vertex AI, Bedrock Converse, and Bedrock Invoke
+      # endpoints. Per-route cells pytest-skip themselves when the matching
+      # provider env vars are absent, so PRs without credentials no-op.
+      - run:
+          name: Run reasoning_effort grid v4 e2e suite
+          command: |
+            mkdir -p test-results
+            uv run --no-sync python -m pytest \
+              tests/test_litellm/reasoning_effort_grid_v4/ \
+              -v \
+              --junitxml=test-results/junit.xml \
+              --durations=20 \
+              -n 4 \
+              --timeout=180 --timeout_method=thread \
+              --retries 2 --retry-delay 5 \
+              --max-worker-restart=5
+          no_output_timeout: 20m
+
       - store_test_results:
           path: test-results
   realtime_translation_testing:
@@ -2619,6 +2661,8 @@ workflows:
           filters: *main_branches
       - llm_translation_testing:
           filters: *main_branches
+      - reasoning_effort_grid_v4_e2e:
+          filters: *main_branches
       - realtime_translation_testing:
           filters: *main_branches
       - agent_testing:

tests/test_litellm/reasoning_effort_grid_v4/conftest.py

@@ -0,0 +1,61 @@
+"""Shared fixtures for the reasoning_effort grid v4 e2e suite."""
+
+import os
+from typing import Any, Dict, List, Optional
+
+import pytest
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+
+
+class _WireBodyCapture(CustomLogger):
+    """Pre-call hook that records the outgoing wire body LiteLLM sends upstream.
+
+    `complete_input_dict` is the fully transformed provider request as set by
+    every provider transformation in `litellm/llms/**`. Capturing it here means
+    a regression anywhere in the transformation chain (strip, rename, drop)
+    surfaces as an assertion failure on the cell that depends on it.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.records: List[Dict[str, Any]] = []
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        self.records.append(
+            {
+                "model": model,
+                "body": kwargs.get("additional_args", {}).get("complete_input_dict"),
+                "api_base": kwargs.get("additional_args", {}).get("api_base"),
+            }
+        )
+
+    async def async_log_pre_api_call(self, model, messages, kwargs):
+        self.log_pre_api_call(model, messages, kwargs)
+
+    def latest(self) -> Optional[Dict[str, Any]]:
+        return self.records[-1] if self.records else None
+
+    def reset(self) -> None:
+        self.records.clear()
+
+
+@pytest.fixture()
+def wire_capture():
+    capture = _WireBodyCapture()
+    previous = list(litellm.callbacks)
+    litellm.callbacks = previous + [capture]
+    try:
+        yield capture
+    finally:
+        litellm.callbacks = previous
+
+
+@pytest.fixture(scope="session")
+def vertex_credentials_path() -> Optional[str]:
+    """Resolve a usable Vertex credentials file path or None."""
+    path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
+    if path and os.path.exists(path):
+        return path
+    return None

tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py

@@ -0,0 +1,301 @@
+"""
+Canonical post-fix expectations for the reasoning_effort grid v4 sweep.
+
+The QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610
+covered 21 (provider x model) combos x 11 effort values (231 cells). The follow-up
+PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs surfaced by that
+sweep. This module encodes the post-fix expectations as a small rule set keyed by
+(model_mode, effort) and per-model capability overrides, then expands them across
+the model x effort matrix per route.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, FrozenSet, List, Optional, Tuple
+
+
+OMIT = object()
+
+
+@dataclass(frozen=True)
+class CellExpectation:
+    """Expected post-fix behavior for a single grid cell."""
+
+    status: int
+    thinking_type: object
+    output_config_effort: object = OMIT
+    thinking_budget_tokens: object = OMIT
+    max_tokens: object = OMIT
+
+
+@dataclass(frozen=True)
+class ModelEntry:
+    alias: str
+    model: str
+    mode: str
+    extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
+    required_env: FrozenSet[str] = field(default_factory=frozenset)
+    caps: FrozenSet[str] = field(default_factory=frozenset)
+
+    def params(self) -> Dict[str, str]:
+        return dict(self.extra_params)
+
+
+EFFORTS: Tuple[str, ...] = (
+    "__omit__",
+    "none",
+    "minimal",
+    "low",
+    "medium",
+    "high",
+    "xhigh",
+    "max",
+    "disabled",
+    "invalid",
+    "",
+)
+
+_BUDGET_TOKENS: Dict[str, int] = {
+    "minimal": 1024,
+    "low": 1024,
+    "medium": 2048,
+    "high": 4096,
+}
+
+_ADAPTIVE_EFFORT_LABEL: Dict[str, str] = {
+    "minimal": "low",
+    "low": "low",
+    "medium": "medium",
+    "high": "high",
+    "xhigh": "xhigh",
+    "max": "max",
+}
+
+_BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""})
+
+
+def expected(model: ModelEntry, effort: str) -> CellExpectation:
+    """Compute the post-fix expected cell for a (model, effort) pair."""
+    if effort in ("__omit__", "none"):
+        if model.mode == "budget":
+            return CellExpectation(status=200, thinking_type=OMIT, max_tokens=8192)
+        return CellExpectation(status=200, thinking_type=OMIT)
+
+    if effort in _BAD_REQUEST_EFFORTS:
+        return CellExpectation(status=400, thinking_type=OMIT)
+
+    if effort in ("xhigh", "max"):
+        cap = f"supports_{effort}_reasoning_effort"
+        if cap not in model.caps:
+            return CellExpectation(status=400, thinking_type=OMIT)
+
+    if model.mode == "adaptive":
+        return CellExpectation(
+            status=200,
+            thinking_type="adaptive",
+            output_config_effort=_ADAPTIVE_EFFORT_LABEL[effort],
+        )
+
+    return CellExpectation(
+        status=200,
+        thinking_type="enabled",
+        thinking_budget_tokens=_BUDGET_TOKENS[effort],
+        max_tokens=8192,
+    )
+
+
+_ANTHROPIC_REQ = frozenset({"ANTHROPIC_API_KEY"})
+_AZURE_FOUNDRY_REQ = frozenset({"AZURE_FOUNDRY_API_BASE", "AZURE_FOUNDRY_API_KEY"})
+_VERTEX_REQ = frozenset({"VERTEX_PROJECT"})
+_BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"})
+
+
+_CAPS_OPUS_4_7: FrozenSet[str] = frozenset(
+    {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"}
+)
+_CAPS_OPUS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"})
+_CAPS_NONE: FrozenSet[str] = frozenset()
+
+
+ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="claude-opus-4-7",
+        model="anthropic/claude-opus-4-7",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="claude-sonnet-4-6",
+        model="anthropic/claude-sonnet-4-6",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_NONE,
+    ),
+    ModelEntry(
+        alias="claude-haiku-4-5",
+        model="anthropic/claude-haiku-4-5",
+        mode="budget",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="azure-claude-opus-4-7",
+        model="azure_ai/claude-opus-4-7",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="azure-claude-opus-4-6",
+        model="azure_ai/claude-opus-4-6",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_OPUS_4_6,
+    ),
+    ModelEntry(
+        alias="azure-claude-sonnet-4-6",
+        model="azure_ai/claude-sonnet-4-6",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_NONE,
+    ),
+    ModelEntry(
+        alias="azure-claude-haiku-4-5",
+        model="azure_ai/claude-haiku-4-5",
+        mode="budget",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="vertex-claude-opus-4-7",
+        model="vertex_ai/claude-opus-4-7",
+        mode="adaptive",
+        extra_params=(("vertex_location", "global"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="vertex-claude-opus-4-6",
+        model="vertex_ai/claude-opus-4-6",
+        mode="adaptive",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_OPUS_4_6,
+    ),
+    ModelEntry(
+        alias="vertex-claude-sonnet-4-6",
+        model="vertex_ai/claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_NONE,
+    ),
+    ModelEntry(
+        alias="vertex-claude-haiku-4-5",
+        model="vertex_ai/claude-haiku-4-5",
+        mode="budget",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-claude-opus-4-7",
+        model="bedrock/converse/us.anthropic.claude-opus-4-7",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-opus-4-6",
+        model="bedrock/converse/us.anthropic.claude-opus-4-6-v1",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_OPUS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-sonnet-4-6",
+        model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-sonnet-4-5",
+        model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+        mode="budget",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-invoke-claude-opus-4-6",
+        model="bedrock/invoke/us.anthropic.claude-opus-4-6-v1",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_OPUS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-invoke-claude-sonnet-4-6",
+        model="bedrock/invoke/us.anthropic.claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+    ModelEntry(
+        alias="bedrock-invoke-claude-opus-4-5",
+        model="bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
+        mode="budget",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_INVOKE_MESSAGES_MODELS: Tuple[ModelEntry, ...] = BEDROCK_INVOKE_CHAT_MODELS
+
+
+@dataclass(frozen=True)
+class Route:
+    name: str
+    models: Tuple[ModelEntry, ...]
+
+
+ROUTES: Tuple[Route, ...] = (
+    Route("anthropic_direct", ANTHROPIC_DIRECT_MODELS),
+    Route("azure_ai", AZURE_AI_MODELS),
+    Route("vertex_ai", VERTEX_AI_MODELS),
+    Route("bedrock_converse", BEDROCK_CONVERSE_MODELS),
+    Route("bedrock_invoke_chat", BEDROCK_INVOKE_CHAT_MODELS),
+    Route("bedrock_invoke_messages", BEDROCK_INVOKE_MESSAGES_MODELS),
+)
+
+
+def all_cells() -> List[Tuple[str, ModelEntry, str, CellExpectation]]:
+    cells: List[Tuple[str, ModelEntry, str, CellExpectation]] = []
+    for route in ROUTES:
+        for model in route.models:
+            for effort in EFFORTS:
+                cells.append((route.name, model, effort, expected(model, effort)))
+    return cells

tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py

@@ -0,0 +1,230 @@
+"""
+End-to-end grid v4 regression suite for reasoning_effort mapping across
+Anthropic-backed routes.
+
+Encodes the 21 (provider x model) x 11 effort matrix (231 cells) from the
+QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610
+that the fix in https://github.com/BerriAI/litellm/pull/27074 was validated
+against. Each cell asserts:
+
+  - Wire body shape captured pre-call (thinking.type, output_config.effort,
+    thinking.budget_tokens, max_tokens) -- the regression signal for silent
+    drops/strips anywhere in the transformation chain.
+  - Status code returned by LiteLLM (200 vs BadRequestError -> 400) -- the
+    regression signal for clean-error vs leaked-500 mappings.
+
+Hits real provider endpoints. Each route is skipped at runtime when its
+required env vars are absent, so PR builds without provider credentials no-op
+gracefully.
+"""
+
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import pytest
+
+import litellm
+from litellm.exceptions import BadRequestError
+
+from .grid_spec import (
+    OMIT,
+    ROUTES,
+    CellExpectation,
+    ModelEntry,
+    all_cells,
+)
+
+
+_PROMPT_MESSAGES: List[Dict[str, str]] = [
+    {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
+]
+
+
+def _required_env_missing(model: ModelEntry) -> Optional[str]:
+    missing = [key for key in model.required_env if not os.environ.get(key)]
+    if missing:
+        return "missing env: " + ", ".join(sorted(missing))
+    return None
+
+
+def _max_tokens_for(model: ModelEntry) -> int:
+    return 200 if model.mode == "adaptive" else 8192
+
+
+def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]:
+    kwargs: Dict[str, Any] = {
+        "model": model.model,
+        "messages": _PROMPT_MESSAGES,
+        "max_tokens": _max_tokens_for(model),
+    }
+    kwargs.update(model.params())
+    if effort != "__omit__":
+        kwargs["reasoning_effort"] = effort
+    if model.model.startswith("vertex_ai/"):
+        kwargs["vertex_project"] = os.environ.get(
+            "VERTEX_PROJECT", "vertex-check-481318"
+        )
+    if model.model.startswith("azure_ai/"):
+        kwargs["api_base"] = os.environ["AZURE_FOUNDRY_API_BASE"]
+        kwargs["api_key"] = os.environ["AZURE_FOUNDRY_API_KEY"]
+    return kwargs
+
+
+def _build_messages_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]:
+    kwargs = _build_completion_kwargs(model, effort)
+    return kwargs
+
+
+def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]:
+    """Return the dict that holds thinking/output_config for a Converse wire body."""
+    return body.get("additionalModelRequestFields", body)
+
+
+def _max_tokens_from_body(body: Dict[str, Any], route_name: str) -> Optional[int]:
+    if route_name == "bedrock_converse":
+        return body.get("inferenceConfig", {}).get("maxTokens")
+    return body.get("max_tokens")
+
+
+def _assert_cell(
+    route_name: str,
+    body: Optional[Dict[str, Any]],
+    status: int,
+    cell: CellExpectation,
+) -> None:
+    assert status == cell.status, f"expected status={cell.status}, got status={status}"
+
+    if cell.status != 200:
+        # Bad-request paths short-circuit before the wire body matters.
+        return
+
+    assert body is not None, "wire body was not captured for a 200-status cell"
+    subbody = _converse_subbody(body) if route_name == "bedrock_converse" else body
+    thinking = subbody.get("thinking")
+    output_config = subbody.get("output_config")
+
+    if cell.thinking_type is OMIT:
+        assert thinking is None, f"expected thinking omitted, got {thinking!r}"
+    else:
+        assert thinking is not None, "expected thinking present, got omit"
+        assert thinking.get("type") == cell.thinking_type, (
+            f"expected thinking.type={cell.thinking_type!r}, "
+            f"got {thinking.get('type')!r}"
+        )
+
+    if cell.output_config_effort is OMIT:
+        assert (
+            output_config is None or "effort" not in output_config
+        ), f"expected output_config.effort omitted, got {output_config!r}"
+    else:
+        assert output_config is not None, (
+            f"expected output_config.effort={cell.output_config_effort!r}, "
+            "got output_config omitted"
+        )
+        assert output_config.get("effort") == cell.output_config_effort, (
+            f"expected output_config.effort={cell.output_config_effort!r}, "
+            f"got {output_config.get('effort')!r}"
+        )
+
+    if cell.thinking_budget_tokens is not OMIT:
+        assert thinking is not None
+        assert thinking.get("budget_tokens") == cell.thinking_budget_tokens, (
+            f"expected thinking.budget_tokens={cell.thinking_budget_tokens!r}, "
+            f"got {thinking.get('budget_tokens')!r}"
+        )
+
+    if cell.max_tokens is not OMIT:
+        wire_max = _max_tokens_from_body(body, route_name)
+        assert (
+            wire_max == cell.max_tokens
+        ), f"expected max_tokens={cell.max_tokens!r}, got {wire_max!r}"
+
+
+_PARAMS: List[Tuple[str, ModelEntry, str, CellExpectation]] = all_cells()
+
+
+def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str:
+    route_name, model, effort, _ = case
+    effort_label = "__empty__" if effort == "" else effort
+    return f"{route_name}-{model.alias}-{effort_label}"
+
+
+_PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS]
+
+
+async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]:
+    kwargs = _build_completion_kwargs(model, effort)
+    try:
+        await litellm.acompletion(**kwargs)
+        return 200, None
+    except BadRequestError as exc:
+        return 400, exc
+    except Exception as exc:
+        return 500, exc
+
+
+async def _call_messages(
+    model: ModelEntry, effort: str
+) -> Tuple[int, Optional[Exception]]:
+    kwargs = _build_messages_kwargs(model, effort)
+    try:
+        await litellm.messages.acreate(**kwargs)
+        return 200, None
+    except BadRequestError as exc:
+        return 400, exc
+    except Exception as exc:
+        return 500, exc
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("route_name", "model", "effort", "cell"), _PARAMS, ids=_PARAM_IDS
+)
+async def test_reasoning_effort_grid_v4(
+    route_name: str,
+    model: ModelEntry,
+    effort: str,
+    cell: CellExpectation,
+    wire_capture,
+) -> None:
+    skip_reason = _required_env_missing(model)
+    if skip_reason:
+        pytest.skip(skip_reason)
+
+    if route_name == "bedrock_invoke_messages":
+        status, exc = await _call_messages(model, effort)
+    else:
+        status, exc = await _call_chat(model, effort)
+
+    record = wire_capture.latest()
+    body = record["body"] if record else None
+
+    try:
+        _assert_cell(route_name, body, status, cell)
+    except AssertionError:
+        if exc is not None:
+            raise AssertionError(
+                f"underlying exception ({type(exc).__name__}): {exc}"
+            ) from None
+        raise
+
+
+def test_grid_v4_cell_count() -> None:
+    """Guard against accidental drops or duplicates in the grid spec."""
+    assert len(_PARAMS) == 21 * 11, (
+        f"expected 231 cells (21 provider x model combos x 11 efforts), "
+        f"got {len(_PARAMS)}"
+    )
+
+
+def test_grid_v4_route_coverage() -> None:
+    """The grid must cover every route the original QA sweep covered."""
+    route_names = {route.name for route in ROUTES}
+    assert route_names == {
+        "anthropic_direct",
+        "azure_ai",
+        "vertex_ai",
+        "bedrock_converse",
+        "bedrock_invoke_chat",
+        "bedrock_invoke_messages",
+    }