diff --git a/litellm/translation/providers/azure/guard.py b/litellm/translation/providers/azure/guard.py index c2dfc0ef71..e9fef684e4 100644 --- a/litellm/translation/providers/azure/guard.py +++ b/litellm/translation/providers/azure/guard.py @@ -22,6 +22,7 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from ...errors import TranslationError from ..openai_compat import unsupported_request_shapes as openai_unsupported_shapes +from ..openai_compat.guard import explicit_stream_false _Raw = Mapping[str, object] @@ -30,6 +31,9 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None: shared = openai_unsupported_shapes(raw) if shared is not None: return shared + stream_false = explicit_stream_false(raw) + if stream_false is not None: + return stream_false reason = _azure_reason(raw) if reason is None: return None @@ -37,8 +41,6 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None: def _azure_reason(raw: _Raw) -> str | None: - if "stream" in raw and raw.get("stream") is False: - return "explicit stream: false (azure keeps the key on the wire; absent-vs-false is lost in the IR)" for field in ("messages", "tools"): if _carries_cache_control(raw.get(field), 0): return ( diff --git a/litellm/translation/providers/compat_sdk/guard.py b/litellm/translation/providers/compat_sdk/guard.py index a63a2c7f40..ef11372272 100644 --- a/litellm/translation/providers/compat_sdk/guard.py +++ b/litellm/translation/providers/compat_sdk/guard.py @@ -1,14 +1,14 @@ """Raw-shape fidelity guard for the SDK-path openai-compat family. -One family-wide arm before the shared openai guard: an explicit -``stream: false``. On this path ``completion()`` forwards the caller's False -into ``get_optional_params`` (non-default against the ``None`` default), it -lands in optional_params, and the SDK serializes the key onto the wire — -while the IR cannot represent absent-vs-false (verified in-process at HEAD; -the same arm the azure and xai guards carry). +The shared explicit ``stream: false`` arm runs first: on this path +``completion()`` forwards the caller's False into ``get_optional_params`` +(non-default against the ``None`` default), it lands in optional_params, +and the SDK serializes the key onto the wire — while the IR cannot +represent absent-vs-false (verified in-process at HEAD; the same arm the +azure and xai guards compose). -The openai guard runs with its full message-``name`` fallback: none of the -family configs strips names (only xai does), so v1 forwards ``name`` +The openai guard then runs with its full message-``name`` fallback: none of +the family configs strips names (only xai does), so v1 forwards ``name`` verbatim on every role. """ @@ -17,6 +17,7 @@ from __future__ import annotations from collections.abc import Mapping from ...errors import TranslationError +from ..openai_compat.guard import explicit_stream_false from ..openai_compat.guard import ( unsupported_request_shapes as openai_unsupported_request_shapes, ) @@ -25,9 +26,4 @@ _Raw = Mapping[str, object] def unsupported_request_shapes(raw: _Raw) -> TranslationError | None: - if "stream" in raw and raw.get("stream") is False: - return TranslationError.of_unsupported( - "explicit stream: false (the SDK path serializes the key onto " - "the wire; absent-vs-false is lost in the IR)" - ) - return openai_unsupported_request_shapes(raw) + return explicit_stream_false(raw) or openai_unsupported_request_shapes(raw) diff --git a/litellm/translation/providers/compat_sdk/params.py b/litellm/translation/providers/compat_sdk/params.py index 93aba6acdd..601a020b28 100644 --- a/litellm/translation/providers/compat_sdk/params.py +++ b/litellm/translation/providers/compat_sdk/params.py @@ -91,6 +91,16 @@ def unsupported_against( note = notes.get(key) if note is not None: return note + if key == "top_k": + # top_k is not an OpenAI param: it never enters + # non_default_params, so _check_valid_arg never sees it and v1 + # drops it WITHOUT drop_params (verified in-process at HEAD; + # verifier-wave1a F6). + return ( + f"top_k on {provider}: not an OpenAI param; v1's " + "get_optional_params silently drops it (no raise, even " + "without drop_params)" + ) return ( f"{key} on {provider}: outside v1's supported list; " "get_optional_params raises UnsupportedParamsError " diff --git a/litellm/translation/providers/openai_compat/guard.py b/litellm/translation/providers/openai_compat/guard.py index ed4c14f3fe..7f049e3698 100644 --- a/litellm/translation/providers/openai_compat/guard.py +++ b/litellm/translation/providers/openai_compat/guard.py @@ -21,6 +21,20 @@ from ...errors import TranslationError _Raw = Mapping[str, object] +def explicit_stream_false(raw: _Raw) -> TranslationError | None: + """The ONE explicit ``stream: false`` arm the azure / xai / compat_sdk + guards compose (critic-wave1a N2): those paths all serialize an + explicitly-sent ``false`` onto the wire while absent-vs-false is lost in + the IR. openai_compat itself deliberately does NOT run it -- the + gap-parity question is integrator scope (wave1a-port.md attack #2).""" + if raw.get("stream") is False: + return TranslationError.of_unsupported( + "explicit stream: false (this path keeps the key on the wire; " + "absent-vs-false is lost in the IR)" + ) + return None + + def unsupported_request_shapes( raw: _Raw, *, name_fallback_user_only: bool = False ) -> TranslationError | None: diff --git a/litellm/translation/providers/xai/guard.py b/litellm/translation/providers/xai/guard.py index 4667ffc458..190ab0f013 100644 --- a/litellm/translation/providers/xai/guard.py +++ b/litellm/translation/providers/xai/guard.py @@ -27,6 +27,7 @@ from typing import cast from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from ...errors import TranslationError +from ..openai_compat.guard import explicit_stream_false from ..openai_compat.guard import ( unsupported_request_shapes as openai_unsupported_request_shapes, ) @@ -56,11 +57,9 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None: "use_xai_oauth: v1 runs an interactive browser PKCE flow inside " "validate_environment (llms/xai/oauth.py); v1 owns it" ) - if "stream" in raw and raw.get("stream") is False: - return TranslationError.of_unsupported( - "explicit stream: false (the xai httpx path keeps the key on the " - "wire; absent-vs-false is lost in the IR)" - ) + stream_false = explicit_stream_false(raw) + if stream_false is not None: + return stream_false reason = _nested_tool_strict_reason(raw) if reason is not None: return TranslationError.of_unsupported(reason) diff --git a/tests/test_litellm/translation/_compat_sdk_corpus.py b/tests/test_litellm/translation/_compat_sdk_corpus.py index 03b0bbe23e..6c21f3429a 100644 --- a/tests/test_litellm/translation/_compat_sdk_corpus.py +++ b/tests/test_litellm/translation/_compat_sdk_corpus.py @@ -16,121 +16,139 @@ remap). """ import copy -from typing import Any, Dict +from collections.abc import Mapping +from types import MappingProxyType +from typing import Literal, NamedTuple from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_optional_params -# One spec per surviving wave-1a provider: the corpus model, which optional -# surfaces the provider serves (drives generated corpus/raise/fallback rows), -# and how max_completion_tokens behaves in v1 ("rename" -> max_tokens, -# "verbatim" -> passes through, "raise" -> outside the supported list). -SPECS: Dict[str, Dict[str, Any]] = { - "together_ai": { - "model": "Qwen/Qwen2.5-72B-Instruct-Turbo", # model-map: supports_function_calling - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "verbatim", - }, - "cerebras": { - "model": "llama3.1-8b", - "tools": True, - "response_format": True, - "parallel_tool_calls": False, - "user": True, - "mct": "rename", - }, - "nvidia_nim": { - "model": "meta/llama3-70b-instruct", # the default-list arm - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "rename", - }, - "lm_studio": { - "model": "qwen2.5-7b-instruct-1m", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "verbatim", - }, - "llamafile": { - "model": "LLaMA_CPP", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "verbatim", - }, - "lambda_ai": { - "model": "llama3.1-70b-instruct-fp8", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "rename", - }, - "nebius": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "rename", - }, - "novita": { - "model": "meta-llama/llama-3.1-8b-instruct", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "verbatim", - }, - "wandb": { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tools": True, - "response_format": True, - "parallel_tool_calls": True, - "user": False, - "mct": "rename", - }, - "featherless_ai": { - "model": "featherless-ai/Qwerky-72B", - "tools": False, - "response_format": False, - "parallel_tool_calls": False, - "user": False, - "mct": "rename", - }, - "nscale": { - "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", - "tools": False, - "response_format": True, - "parallel_tool_calls": False, - "user": False, - "mct": "raise", - }, - "hyperbolic": { - "model": "meta-llama/Meta-Llama-3-70B-Instruct", - "tools": True, - "response_format": True, - "parallel_tool_calls": False, - "user": True, - "mct": "raise", - }, - "volcengine": { - "model": "doubao-pro-32k-241215", - "tools": True, - "response_format": False, - "parallel_tool_calls": False, - "user": False, - "mct": "rename", - }, -} +_Case = dict[str, object] + + +class CompatSpec(NamedTuple): + """One row per surviving wave-1a provider: the corpus model, which + optional surfaces the provider serves (drives generated + corpus/raise/fallback rows), and how max_completion_tokens behaves in + v1 ("rename" -> max_tokens, "verbatim" -> passes through, "raise" -> + outside the supported list).""" + + model: str + tools: bool + response_format: bool + parallel_tool_calls: bool + user: bool + mct: Literal["rename", "verbatim", "raise"] + + +SPECS: Mapping[str, CompatSpec] = MappingProxyType( + { + "together_ai": CompatSpec( + model="Qwen/Qwen2.5-72B-Instruct-Turbo", # map: supports_function_calling + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="verbatim", + ), + "cerebras": CompatSpec( + model="llama3.1-8b", + tools=True, + response_format=True, + parallel_tool_calls=False, + user=True, + mct="rename", + ), + "nvidia_nim": CompatSpec( + model="meta/llama3-70b-instruct", # the default-list arm + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="rename", + ), + "lm_studio": CompatSpec( + model="qwen2.5-7b-instruct-1m", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="verbatim", + ), + "llamafile": CompatSpec( + model="LLaMA_CPP", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="verbatim", + ), + "lambda_ai": CompatSpec( + model="llama3.1-70b-instruct-fp8", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="rename", + ), + "nebius": CompatSpec( + model="meta-llama/Meta-Llama-3.1-70B-Instruct", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="rename", + ), + "novita": CompatSpec( + model="meta-llama/llama-3.1-8b-instruct", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="verbatim", + ), + "wandb": CompatSpec( + model="meta-llama/Llama-3.1-8B-Instruct", + tools=True, + response_format=True, + parallel_tool_calls=True, + user=False, + mct="rename", + ), + "featherless_ai": CompatSpec( + model="featherless-ai/Qwerky-72B", + tools=False, + response_format=False, + parallel_tool_calls=False, + user=False, + mct="rename", + ), + "nscale": CompatSpec( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + tools=False, + response_format=True, + parallel_tool_calls=False, + user=False, + mct="raise", + ), + "hyperbolic": CompatSpec( + model="meta-llama/Meta-Llama-3-70B-Instruct", + tools=True, + response_format=True, + parallel_tool_calls=False, + user=True, + mct="raise", + ), + "volcengine": CompatSpec( + model="doubao-pro-32k-241215", + tools=True, + response_format=False, + parallel_tool_calls=False, + user=False, + mct="rename", + ), + } +) PROVIDERS = tuple(sorted(SPECS)) @@ -154,7 +172,7 @@ def provider_config(provider: str, model: str): ) -def run_v1_request_transform(provider: str, case: Dict[str, Any]) -> Dict[str, Any]: +def run_v1_request_transform(provider: str, case: _Case) -> dict: request = copy.deepcopy(case) model = request.pop("model") messages = request.pop("messages") @@ -176,13 +194,13 @@ def run_v1_request_transform(provider: str, case: Dict[str, Any]) -> Dict[str, A ) -def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]: +def corpus_for(provider: str) -> dict[str, _Case]: """The generated served corpus: every row here must be byte-identical (normalized JSON) between v1-in-process and v2.""" spec = SPECS[provider] - model = spec["model"] + model = spec.model user_msg = [{"role": "user", "content": "Hello, world"}] - cases: Dict[str, Dict[str, Any]] = { + cases: dict[str, _Case] = { "text": {"model": model, "messages": user_msg}, "system_and_sampling": { "model": model, @@ -202,13 +220,13 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]: "messages": user_msg, }, } - if spec["mct"] in ("rename", "verbatim"): + if spec.mct in ("rename", "verbatim"): cases["max_completion_tokens"] = { "model": model, "max_completion_tokens": 128, "messages": user_msg, } - if spec["tools"]: + if spec.tools: cases["tools_auto"] = { "model": model, "tools": [WEATHER_TOOL], @@ -243,7 +261,7 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]: {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, ], } - if spec["response_format"]: + if spec.response_format: cases["response_format_json_object"] = { "model": model, "response_format": {"type": "json_object"}, @@ -266,13 +284,13 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]: }, "messages": user_msg, } - if spec["parallel_tool_calls"]: + if spec.parallel_tool_calls: cases["parallel_tool_calls_false"] = { "model": model, "tools": [WEATHER_TOOL], "parallel_tool_calls": False, "messages": [{"role": "user", "content": "Weather in Paris and Rome?"}], } - if spec["user"]: + if spec.user: cases["user_param"] = {"model": model, "user": "u-1", "messages": user_msg} return cases diff --git a/tests/test_litellm/translation/generate_differential_report.py b/tests/test_litellm/translation/generate_differential_report.py index 1f802fda3b..2033259771 100644 --- a/tests/test_litellm/translation/generate_differential_report.py +++ b/tests/test_litellm/translation/generate_differential_report.py @@ -328,7 +328,7 @@ def _compat_sdk_rows(lines: list) -> int: ] for provider, name in resp._rows(): raw = resp._RESPONSES[name] - preset = f"{provider}/{corpus.SPECS[provider]['model']}" + preset = f"{provider}/{corpus.SPECS[provider].model}" same = resp._norm(resp._v2_model_response(provider, raw, preset)) == resp._norm( resp._v1_model_response(raw, preset) ) diff --git a/tests/test_litellm/translation/test_differential_compat_sdk_request.py b/tests/test_litellm/translation/test_differential_compat_sdk_request.py index e7a51534bf..230f55e945 100644 --- a/tests/test_litellm/translation/test_differential_compat_sdk_request.py +++ b/tests/test_litellm/translation/test_differential_compat_sdk_request.py @@ -41,7 +41,7 @@ _TOOLS = [WEATHER_TOOL] def _m(provider: str) -> str: - return SPECS[provider]["model"] + return SPECS[provider].model # Rows where v1 RAISES UnsupportedParamsError (the supported-list gate); v2 @@ -51,19 +51,19 @@ def _m(provider: str) -> str: V1_RAISES = {} for _p in PROVIDERS: _spec = SPECS[_p] - if _spec["mct"] == "raise": + if _spec.mct == "raise": V1_RAISES[f"{_p}:max_completion_tokens"] = ( _p, {"model": _m(_p), "max_completion_tokens": 128, "messages": _USER}, "max_completion_tokens", ) - if not _spec["tools"]: + if not _spec.tools: V1_RAISES[f"{_p}:tools"] = ( _p, {"model": _m(_p), "tools": _TOOLS, "messages": _USER}, "tools", ) - if not _spec["response_format"]: + if not _spec.response_format: V1_RAISES[f"{_p}:response_format"] = ( _p, { @@ -73,7 +73,7 @@ for _p in PROVIDERS: }, "response_format", ) - if not _spec["parallel_tool_calls"]: + if not _spec.parallel_tool_calls: V1_RAISES[f"{_p}:parallel_tool_calls"] = ( _p, {"model": _m(_p), "parallel_tool_calls": False, "messages": _USER}, @@ -231,7 +231,7 @@ for _p in PROVIDERS: {"model": _m(_p), "seed": 42, "messages": _USER}, "seed", ) - if not SPECS[_p]["user"]: + if not SPECS[_p].user: EXPECTED_FALLBACKS[f"{_p}:user_model_list_gate"] = ( _p, {"model": _m(_p), "user": "u-1", "messages": _USER}, @@ -268,7 +268,7 @@ EXPECTED_FALLBACKS.update( ) -def _v2(provider: str, case: dict): +def _v2(provider: str, case: dict[str, object]): return translate_chat_request(copy.deepcopy(case), provider, build_real_deps()) @@ -375,14 +375,14 @@ _MIRROR_KEYS = ( _RF_NAME_GATED = ("gpt-4", "gpt-3.5-turbo-16k") -def _base_family_allowed(model: str) -> frozenset: +def _base_family_allowed(model: str) -> frozenset[str]: allowed = csp._BASE_LIST if model in _RF_NAME_GATED: return allowed - {"response_format"} return allowed -def _v2_allowed(provider: str, model: str, deps) -> frozenset: +def _v2_allowed(provider: str, model: str, deps) -> frozenset[str]: """The per-model allowed set, re-derived from the SAME csp.ALLOWED table serialization reads, with the three per-model narrowings the gates apply (together capability fork, nvidia_nim static table, the base @@ -450,7 +450,7 @@ def test_supported_list_mirrors_track_v1_at_head(provider: str) -> None: ) for key in _MIRROR_KEYS: assert (key in allowed) == (key in supported), (provider, model, key) - if SPECS[provider]["user"]: + if SPECS[provider].user: assert "user" in supported, (provider, model) if provider == "cerebras": assert csp.supports_cerebras_reasoning(model, deps) == ( @@ -483,9 +483,7 @@ def test_capability_prefix_is_load_bearing() -> None: ) == litellm.supports_reasoning(model="qwen-3-32b", custom_llm_provider="cerebras") -@pytest.mark.parametrize( - "provider", [p for p in PROVIDERS if SPECS[p]["mct"] == "rename"] -) +@pytest.mark.parametrize("provider", [p for p in PROVIDERS if SPECS[p].mct == "rename"]) def test_mct_rename_matches_v1(provider: str) -> None: """The renamed key must be exactly what v1's map emits (max_tokens), so the rename flag can never silently disagree with the v1 config.""" @@ -499,7 +497,7 @@ def test_mct_rename_matches_v1(provider: str) -> None: @pytest.mark.parametrize( - "provider", [p for p in PROVIDERS if SPECS[p]["mct"] == "verbatim"] + "provider", [p for p in PROVIDERS if SPECS[p].mct == "verbatim"] ) def test_mct_verbatim_matches_v1(provider: str) -> None: case = {"model": _m(provider), "max_completion_tokens": 33, "messages": _USER} @@ -511,6 +509,20 @@ def test_mct_verbatim_matches_v1(provider: str) -> None: assert "max_tokens" not in result.ok +def test_top_k_fallback_reason_matches_v1_silent_drop() -> None: + """v1 silently drops top_k WITHOUT drop_params (it is not an OpenAI + param, so _check_valid_arg never sees it) — the fallback reason must say + so instead of claiming an UnsupportedParamsError raise (verifier-wave1a + F6). Pinned in-process: v1 serves the request with top_k gone.""" + case = {"model": _m("lambda_ai"), "top_k": 3, "messages": _USER} + result = _v2("lambda_ai", case) + assert result.is_error() + assert "silently drops it" in result.error.summary, result.error.summary + assert "UnsupportedParamsError" not in result.error.summary + v1 = run_v1_request_transform("lambda_ai", case) + assert "top_k" not in v1 + + def test_together_text_response_format_dropped_like_v1() -> None: case = { "model": _m("together_ai"), diff --git a/tests/test_litellm/translation/test_differential_compat_sdk_response.py b/tests/test_litellm/translation/test_differential_compat_sdk_response.py index efc5e077ca..4eb04bb4dc 100644 --- a/tests/test_litellm/translation/test_differential_compat_sdk_response.py +++ b/tests/test_litellm/translation/test_differential_compat_sdk_response.py @@ -37,7 +37,7 @@ _RESPONSE_ROWS = ( def _request_for(provider: str) -> dict: return { - "model": SPECS[provider]["model"], + "model": SPECS[provider].model, "messages": [{"role": "user", "content": "hi"}], } @@ -74,7 +74,7 @@ def test_preset_reprefix_matches_v1(provider: str, name: str, frozen_ambient) -> """SDK-path preset: {provider}/{request model} in, {provider}/{wire model} out, byte-identical dumps both sides.""" raw = _RESPONSES[name] - preset = f"{provider}/{SPECS[provider]['model']}" + preset = f"{provider}/{SPECS[provider].model}" v1 = _v1_model_response(raw, preset) v2 = _v2_model_response(provider, raw, preset) assert _norm(v2) == _norm(v1) @@ -87,7 +87,7 @@ def test_preset_survives_when_wire_model_missing(provider: str, frozen_ambient) """cdr's elif arm needs a non-None wire model; without one the preset {provider}/{request model} survives verbatim on both sides.""" raw = {k: v for k, v in _RESPONSES["text"].items() if k != "model"} - preset = f"{provider}/{SPECS[provider]['model']}" + preset = f"{provider}/{SPECS[provider].model}" v1 = _v1_model_response(raw, preset) v2 = _v2_model_response(provider, raw, preset) assert _norm(v2) == _norm(v1)