diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a5b6da77f..3dd1221c12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -576,6 +576,48 @@ jobs: no_output_timeout: 15m # Store test results + - store_test_results: + path: test-results + reasoning_effort_grid_v4_e2e: + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Grid v4 exercises reasoning_effort mapping against real Anthropic, + # Azure AI Foundry, Vertex AI, Bedrock Converse, and Bedrock Invoke + # endpoints. Per-route cells pytest-skip themselves when the matching + # provider env vars are absent, so PRs without credentials no-op. + - run: + name: Run reasoning_effort grid v4 e2e suite + command: | + mkdir -p test-results + uv run --no-sync python -m pytest \ + tests/test_litellm/reasoning_effort_grid_v4/ \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=180 --timeout_method=thread \ + --retries 2 --retry-delay 5 \ + --max-worker-restart=5 + no_output_timeout: 20m + - store_test_results: path: test-results realtime_translation_testing: @@ -2619,6 +2661,8 @@ workflows: filters: *main_branches - llm_translation_testing: filters: *main_branches + - reasoning_effort_grid_v4_e2e: + filters: *main_branches - realtime_translation_testing: filters: *main_branches - agent_testing: diff --git a/tests/test_litellm/reasoning_effort_grid_v4/__init__.py b/tests/test_litellm/reasoning_effort_grid_v4/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py new file mode 100644 index 0000000000..5e1a4e0e97 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py @@ -0,0 +1,61 @@ +"""Shared fixtures for the reasoning_effort grid v4 e2e suite.""" + +import os +from typing import Any, Dict, List, Optional + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger + + +class _WireBodyCapture(CustomLogger): + """Pre-call hook that records the outgoing wire body LiteLLM sends upstream. + + `complete_input_dict` is the fully transformed provider request as set by + every provider transformation in `litellm/llms/**`. Capturing it here means + a regression anywhere in the transformation chain (strip, rename, drop) + surfaces as an assertion failure on the cell that depends on it. + """ + + def __init__(self) -> None: + super().__init__() + self.records: List[Dict[str, Any]] = [] + + def log_pre_api_call(self, model, messages, kwargs): + self.records.append( + { + "model": model, + "body": kwargs.get("additional_args", {}).get("complete_input_dict"), + "api_base": kwargs.get("additional_args", {}).get("api_base"), + } + ) + + async def async_log_pre_api_call(self, model, messages, kwargs): + self.log_pre_api_call(model, messages, kwargs) + + def latest(self) -> Optional[Dict[str, Any]]: + return self.records[-1] if self.records else None + + def reset(self) -> None: + self.records.clear() + + +@pytest.fixture() +def wire_capture(): + capture = _WireBodyCapture() + previous = list(litellm.callbacks) + litellm.callbacks = previous + [capture] + try: + yield capture + finally: + litellm.callbacks = previous + + +@pytest.fixture(scope="session") +def vertex_credentials_path() -> Optional[str]: + """Resolve a usable Vertex credentials file path or None.""" + path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if path and os.path.exists(path): + return path + return None diff --git a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py new file mode 100644 index 0000000000..76243fe227 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py @@ -0,0 +1,301 @@ +""" +Canonical post-fix expectations for the reasoning_effort grid v4 sweep. + +The QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 +covered 21 (provider x model) combos x 11 effort values (231 cells). The follow-up +PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs surfaced by that +sweep. This module encodes the post-fix expectations as a small rule set keyed by +(model_mode, effort) and per-model capability overrides, then expands them across +the model x effort matrix per route. +""" + +from dataclasses import dataclass, field +from typing import Dict, FrozenSet, List, Optional, Tuple + + +OMIT = object() + + +@dataclass(frozen=True) +class CellExpectation: + """Expected post-fix behavior for a single grid cell.""" + + status: int + thinking_type: object + output_config_effort: object = OMIT + thinking_budget_tokens: object = OMIT + max_tokens: object = OMIT + + +@dataclass(frozen=True) +class ModelEntry: + alias: str + model: str + mode: str + extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple) + required_env: FrozenSet[str] = field(default_factory=frozenset) + caps: FrozenSet[str] = field(default_factory=frozenset) + + def params(self) -> Dict[str, str]: + return dict(self.extra_params) + + +EFFORTS: Tuple[str, ...] = ( + "__omit__", + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + "disabled", + "invalid", + "", +) + +_BUDGET_TOKENS: Dict[str, int] = { + "minimal": 1024, + "low": 1024, + "medium": 2048, + "high": 4096, +} + +_ADAPTIVE_EFFORT_LABEL: Dict[str, str] = { + "minimal": "low", + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "xhigh", + "max": "max", +} + +_BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""}) + + +def expected(model: ModelEntry, effort: str) -> CellExpectation: + """Compute the post-fix expected cell for a (model, effort) pair.""" + if effort in ("__omit__", "none"): + if model.mode == "budget": + return CellExpectation(status=200, thinking_type=OMIT, max_tokens=8192) + return CellExpectation(status=200, thinking_type=OMIT) + + if effort in _BAD_REQUEST_EFFORTS: + return CellExpectation(status=400, thinking_type=OMIT) + + if effort in ("xhigh", "max"): + cap = f"supports_{effort}_reasoning_effort" + if cap not in model.caps: + return CellExpectation(status=400, thinking_type=OMIT) + + if model.mode == "adaptive": + return CellExpectation( + status=200, + thinking_type="adaptive", + output_config_effort=_ADAPTIVE_EFFORT_LABEL[effort], + ) + + return CellExpectation( + status=200, + thinking_type="enabled", + thinking_budget_tokens=_BUDGET_TOKENS[effort], + max_tokens=8192, + ) + + +_ANTHROPIC_REQ = frozenset({"ANTHROPIC_API_KEY"}) +_AZURE_FOUNDRY_REQ = frozenset({"AZURE_FOUNDRY_API_BASE", "AZURE_FOUNDRY_API_KEY"}) +_VERTEX_REQ = frozenset({"VERTEX_PROJECT"}) +_BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}) + + +_CAPS_OPUS_4_7: FrozenSet[str] = frozenset( + {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"} +) +_CAPS_OPUS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"}) +_CAPS_NONE: FrozenSet[str] = frozenset() + + +ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="claude-opus-4-7", + model="anthropic/claude-opus-4-7", + mode="adaptive", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="claude-sonnet-4-6", + model="anthropic/claude-sonnet-4-6", + mode="adaptive", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="claude-haiku-4-5", + model="anthropic/claude-haiku-4-5", + mode="budget", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_NONE, + ), +) + + +AZURE_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="azure-claude-opus-4-7", + model="azure_ai/claude-opus-4-7", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="azure-claude-opus-4-6", + model="azure_ai/claude-opus-4-6", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="azure-claude-sonnet-4-6", + model="azure_ai/claude-sonnet-4-6", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="azure-claude-haiku-4-5", + model="azure_ai/claude-haiku-4-5", + mode="budget", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_NONE, + ), +) + + +VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="vertex-claude-opus-4-7", + model="vertex_ai/claude-opus-4-7", + mode="adaptive", + extra_params=(("vertex_location", "global"),), + required_env=_VERTEX_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="vertex-claude-opus-4-6", + model="vertex_ai/claude-opus-4-6", + mode="adaptive", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="vertex-claude-sonnet-4-6", + model="vertex_ai/claude-sonnet-4-6", + mode="adaptive", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="vertex-claude-haiku-4-5", + model="vertex_ai/claude-haiku-4-5", + mode="budget", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="bedrock-claude-opus-4-7", + model="bedrock/converse/us.anthropic.claude-opus-4-7", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="bedrock-claude-opus-4-6", + model="bedrock/converse/us.anthropic.claude-opus-4-6-v1", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="bedrock-claude-sonnet-4-6", + model="bedrock/converse/us.anthropic.claude-sonnet-4-6", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="bedrock-claude-sonnet-4-5", + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + mode="budget", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="bedrock-invoke-claude-opus-4-6", + model="bedrock/invoke/us.anthropic.claude-opus-4-6-v1", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="bedrock-invoke-claude-sonnet-4-6", + model="bedrock/invoke/us.anthropic.claude-sonnet-4-6", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="bedrock-invoke-claude-opus-4-5", + model="bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", + mode="budget", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_INVOKE_MESSAGES_MODELS: Tuple[ModelEntry, ...] = BEDROCK_INVOKE_CHAT_MODELS + + +@dataclass(frozen=True) +class Route: + name: str + models: Tuple[ModelEntry, ...] + + +ROUTES: Tuple[Route, ...] = ( + Route("anthropic_direct", ANTHROPIC_DIRECT_MODELS), + Route("azure_ai", AZURE_AI_MODELS), + Route("vertex_ai", VERTEX_AI_MODELS), + Route("bedrock_converse", BEDROCK_CONVERSE_MODELS), + Route("bedrock_invoke_chat", BEDROCK_INVOKE_CHAT_MODELS), + Route("bedrock_invoke_messages", BEDROCK_INVOKE_MESSAGES_MODELS), +) + + +def all_cells() -> List[Tuple[str, ModelEntry, str, CellExpectation]]: + cells: List[Tuple[str, ModelEntry, str, CellExpectation]] = [] + for route in ROUTES: + for model in route.models: + for effort in EFFORTS: + cells.append((route.name, model, effort, expected(model, effort))) + return cells diff --git a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py new file mode 100644 index 0000000000..9eec572988 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py @@ -0,0 +1,230 @@ +""" +End-to-end grid v4 regression suite for reasoning_effort mapping across +Anthropic-backed routes. + +Encodes the 21 (provider x model) x 11 effort matrix (231 cells) from the +QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 +that the fix in https://github.com/BerriAI/litellm/pull/27074 was validated +against. Each cell asserts: + + - Wire body shape captured pre-call (thinking.type, output_config.effort, + thinking.budget_tokens, max_tokens) -- the regression signal for silent + drops/strips anywhere in the transformation chain. + - Status code returned by LiteLLM (200 vs BadRequestError -> 400) -- the + regression signal for clean-error vs leaked-500 mappings. + +Hits real provider endpoints. Each route is skipped at runtime when its +required env vars are absent, so PR builds without provider credentials no-op +gracefully. +""" + +import os +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +import litellm +from litellm.exceptions import BadRequestError + +from .grid_spec import ( + OMIT, + ROUTES, + CellExpectation, + ModelEntry, + all_cells, +) + + +_PROMPT_MESSAGES: List[Dict[str, str]] = [ + {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."} +] + + +def _required_env_missing(model: ModelEntry) -> Optional[str]: + missing = [key for key in model.required_env if not os.environ.get(key)] + if missing: + return "missing env: " + ", ".join(sorted(missing)) + return None + + +def _max_tokens_for(model: ModelEntry) -> int: + return 200 if model.mode == "adaptive" else 8192 + + +def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: + kwargs: Dict[str, Any] = { + "model": model.model, + "messages": _PROMPT_MESSAGES, + "max_tokens": _max_tokens_for(model), + } + kwargs.update(model.params()) + if effort != "__omit__": + kwargs["reasoning_effort"] = effort + if model.model.startswith("vertex_ai/"): + kwargs["vertex_project"] = os.environ.get( + "VERTEX_PROJECT", "vertex-check-481318" + ) + if model.model.startswith("azure_ai/"): + kwargs["api_base"] = os.environ["AZURE_FOUNDRY_API_BASE"] + kwargs["api_key"] = os.environ["AZURE_FOUNDRY_API_KEY"] + return kwargs + + +def _build_messages_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: + kwargs = _build_completion_kwargs(model, effort) + return kwargs + + +def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]: + """Return the dict that holds thinking/output_config for a Converse wire body.""" + return body.get("additionalModelRequestFields", body) + + +def _max_tokens_from_body(body: Dict[str, Any], route_name: str) -> Optional[int]: + if route_name == "bedrock_converse": + return body.get("inferenceConfig", {}).get("maxTokens") + return body.get("max_tokens") + + +def _assert_cell( + route_name: str, + body: Optional[Dict[str, Any]], + status: int, + cell: CellExpectation, +) -> None: + assert status == cell.status, f"expected status={cell.status}, got status={status}" + + if cell.status != 200: + # Bad-request paths short-circuit before the wire body matters. + return + + assert body is not None, "wire body was not captured for a 200-status cell" + subbody = _converse_subbody(body) if route_name == "bedrock_converse" else body + thinking = subbody.get("thinking") + output_config = subbody.get("output_config") + + if cell.thinking_type is OMIT: + assert thinking is None, f"expected thinking omitted, got {thinking!r}" + else: + assert thinking is not None, "expected thinking present, got omit" + assert thinking.get("type") == cell.thinking_type, ( + f"expected thinking.type={cell.thinking_type!r}, " + f"got {thinking.get('type')!r}" + ) + + if cell.output_config_effort is OMIT: + assert ( + output_config is None or "effort" not in output_config + ), f"expected output_config.effort omitted, got {output_config!r}" + else: + assert output_config is not None, ( + f"expected output_config.effort={cell.output_config_effort!r}, " + "got output_config omitted" + ) + assert output_config.get("effort") == cell.output_config_effort, ( + f"expected output_config.effort={cell.output_config_effort!r}, " + f"got {output_config.get('effort')!r}" + ) + + if cell.thinking_budget_tokens is not OMIT: + assert thinking is not None + assert thinking.get("budget_tokens") == cell.thinking_budget_tokens, ( + f"expected thinking.budget_tokens={cell.thinking_budget_tokens!r}, " + f"got {thinking.get('budget_tokens')!r}" + ) + + if cell.max_tokens is not OMIT: + wire_max = _max_tokens_from_body(body, route_name) + assert ( + wire_max == cell.max_tokens + ), f"expected max_tokens={cell.max_tokens!r}, got {wire_max!r}" + + +_PARAMS: List[Tuple[str, ModelEntry, str, CellExpectation]] = all_cells() + + +def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str: + route_name, model, effort, _ = case + effort_label = "__empty__" if effort == "" else effort + return f"{route_name}-{model.alias}-{effort_label}" + + +_PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS] + + +async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]: + kwargs = _build_completion_kwargs(model, effort) + try: + await litellm.acompletion(**kwargs) + return 200, None + except BadRequestError as exc: + return 400, exc + except Exception as exc: + return 500, exc + + +async def _call_messages( + model: ModelEntry, effort: str +) -> Tuple[int, Optional[Exception]]: + kwargs = _build_messages_kwargs(model, effort) + try: + await litellm.messages.acreate(**kwargs) + return 200, None + except BadRequestError as exc: + return 400, exc + except Exception as exc: + return 500, exc + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route_name", "model", "effort", "cell"), _PARAMS, ids=_PARAM_IDS +) +async def test_reasoning_effort_grid_v4( + route_name: str, + model: ModelEntry, + effort: str, + cell: CellExpectation, + wire_capture, +) -> None: + skip_reason = _required_env_missing(model) + if skip_reason: + pytest.skip(skip_reason) + + if route_name == "bedrock_invoke_messages": + status, exc = await _call_messages(model, effort) + else: + status, exc = await _call_chat(model, effort) + + record = wire_capture.latest() + body = record["body"] if record else None + + try: + _assert_cell(route_name, body, status, cell) + except AssertionError: + if exc is not None: + raise AssertionError( + f"underlying exception ({type(exc).__name__}): {exc}" + ) from None + raise + + +def test_grid_v4_cell_count() -> None: + """Guard against accidental drops or duplicates in the grid spec.""" + assert len(_PARAMS) == 21 * 11, ( + f"expected 231 cells (21 provider x model combos x 11 efforts), " + f"got {len(_PARAMS)}" + ) + + +def test_grid_v4_route_coverage() -> None: + """The grid must cover every route the original QA sweep covered.""" + route_names = {route.name for route in ROUTES} + assert route_names == { + "anthropic_direct", + "azure_ai", + "vertex_ai", + "bedrock_converse", + "bedrock_invoke_chat", + "bedrock_invoke_messages", + }