From e29ea53c31fe41fb16e69f640d2d05bb16cc2dc6 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 16 May 2026 07:32:25 +0000
Subject: [PATCH] fix(reasoning_effort_grid): classify status by exception
 status_code, not class

The anthropic_messages route wraps client-side BadRequestError as
AnthropicError (a BaseLLMException subclass) with status_code=400, so
"except BadRequestError" missed those cells and they fell through to the
generic Exception arm, returning 500 instead of the expected 400.

Replace the isinstance-on-BadRequestError check with a tiny classifier
that prefers BadRequestError membership, then falls back to the exception's
status_code attribute (set by every BaseLLMException subclass), then 500.
Apply to both _call_chat and _call_messages for consistency.

Fixes the 13 CircleCI llm_translation_testing failures on
bedrock_invoke_messages cells where the effort was disabled / invalid /
empty / xhigh-on-unsupported / max-on-unsupported.
---
 .../test_reasoning_effort_grid.py             | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
index 43f0fa1dbe..a815eb861e 100644
--- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
+++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
@@ -156,15 +156,30 @@ def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str:
 _PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS]
 
 
+def _classify_status(exc: Exception) -> int:
+    """Map an exception to the HTTP status the QA grid would have observed.
+
+    The Anthropic Messages route (litellm.anthropic_messages) wraps client-side
+    BadRequestError as ``AnthropicError`` with ``status_code=400`` rather than
+    re-raising the BadRequestError class directly, so isinstance() alone misses
+    those cells. Read the ``status_code`` attribute when present (set by every
+    BaseLLMException subclass) and fall through to 500 otherwise.
+    """
+    if isinstance(exc, BadRequestError):
+        return 400
+    code = getattr(exc, "status_code", None)
+    if isinstance(code, int):
+        return code
+    return 500
+
+
 async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]:
     kwargs = _build_completion_kwargs(model, effort)
     try:
         await litellm.acompletion(**kwargs)
         return 200, None
-    except BadRequestError as exc:
-        return 400, exc
     except Exception as exc:
-        return 500, exc
+        return _classify_status(exc), exc
 
 
 async def _call_messages(
@@ -174,10 +189,8 @@ async def _call_messages(
     try:
         await litellm.anthropic_messages(**kwargs)
         return 200, None
-    except BadRequestError as exc:
-        return 400, exc
     except Exception as exc:
-        return 500, exc
+        return _classify_status(exc), exc
 
 
 @pytest.mark.asyncio