tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-15 16:24:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(translation): take critic-wave1a NITs N2, N3 and verifier-wave1a F6

Browse Source 
N2: the explicit stream:false arm existed as three byte-equivalent copies
(azure, xai, compat_sdk) -- the parse_line NIT-to-MAJOR trajectory
restarted. Extracted as openai_compat.guard.explicit_stream_false and
composed from all three (the redundant 'stream in raw' conjunct dies with
it); openai_compat itself still deliberately lacks the arm, the gap-parity
question stays integrator scope and is now written at the shared function.

N3: the corpus SPECS table is a frozen Mapping of CompatSpec NamedTuples
(mct is a Literal) instead of the fourth Dict[str, Any] copy of the
corpus-helper house style; the stringly spec['mct'] comparisons are gone
and case payloads are typed dict[str, object].

F6: the generic fallback reason claimed get_optional_params raises
UnsupportedParamsError for top_k, but v1 silently drops top_k WITHOUT
drop_params (not an OpenAI param; _check_valid_arg never sees it). The
reason now states the silent drop, pinned in-process: v1 serves the
request with top_k gone.
This commit is contained in:
mateo-berri
2026-06-12 10:47:05 +00:00
parent 5955754c57
commit c6dfb86247
9 changed files with 210 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/translation/providers/azure/guard.py
+4 -2
View File
@@ -22,6 +22,7 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from ...errors import TranslationError
from ..openai_compat import unsupported_request_shapes as openai_unsupported_shapes
from ..openai_compat.guard import explicit_stream_false
_Raw = Mapping[str, object]
@@ -30,6 +31,9 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None:
shared = openai_unsupported_shapes(raw)
if shared is not None:
return shared
stream_false = explicit_stream_false(raw)
if stream_false is not None:
return stream_false
reason = _azure_reason(raw)
if reason is None:
return None
@@ -37,8 +41,6 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None:
def _azure_reason(raw: _Raw) -> str | None:
if "stream" in raw and raw.get("stream") is False:
return "explicit stream: false (azure keeps the key on the wire; absent-vs-false is lost in the IR)"
for field in ("messages", "tools"):
if _carries_cache_control(raw.get(field), 0):
return (
litellm/translation/providers/compat_sdk/guard.py
+10 -14
View File
@@ -1,14 +1,14 @@
"""Raw-shape fidelity guard for the SDK-path openai-compat family.
One family-wide arm before the shared openai guard: an explicit
``stream: false``. On this path ``completion()`` forwards the caller's False
into ``get_optional_params`` (non-default against the ``None`` default), it
lands in optional_params, and the SDK serializes the key onto the wire —
while the IR cannot represent absent-vs-false (verified in-process at HEAD;
the same arm the azure and xai guards carry).
The shared explicit ``stream: false`` arm runs first: on this path
``completion()`` forwards the caller's False into ``get_optional_params``
(non-default against the ``None`` default), it lands in optional_params,
and the SDK serializes the key onto the wire — while the IR cannot
represent absent-vs-false (verified in-process at HEAD; the same arm the
azure and xai guards compose).
The openai guard runs with its full message-``name`` fallback: none of the
family configs strips names (only xai does), so v1 forwards ``name``
The openai guard then runs with its full message-``name`` fallback: none of
the family configs strips names (only xai does), so v1 forwards ``name``
verbatim on every role.
"""
@@ -17,6 +17,7 @@ from __future__ import annotations
from collections.abc import Mapping
from ...errors import TranslationError
from ..openai_compat.guard import explicit_stream_false
from ..openai_compat.guard import (
unsupported_request_shapes as openai_unsupported_request_shapes,
)
@@ -25,9 +26,4 @@ _Raw = Mapping[str, object]
def unsupported_request_shapes(raw: _Raw) -> TranslationError | None:
if "stream" in raw and raw.get("stream") is False:
return TranslationError.of_unsupported(
"explicit stream: false (the SDK path serializes the key onto "
"the wire; absent-vs-false is lost in the IR)"
)
return openai_unsupported_request_shapes(raw)
return explicit_stream_false(raw) or openai_unsupported_request_shapes(raw)
litellm/translation/providers/compat_sdk/params.py
+10
View File
@@ -91,6 +91,16 @@ def unsupported_against(
note = notes.get(key)
if note is not None:
return note
if key == "top_k":
# top_k is not an OpenAI param: it never enters
# non_default_params, so _check_valid_arg never sees it and v1
# drops it WITHOUT drop_params (verified in-process at HEAD;
# verifier-wave1a F6).
return (
f"top_k on {provider}: not an OpenAI param; v1's "
"get_optional_params silently drops it (no raise, even "
"without drop_params)"
)
return (
f"{key} on {provider}: outside v1's supported list; "
"get_optional_params raises UnsupportedParamsError "
litellm/translation/providers/openai_compat/guard.py
+14
View File
@@ -21,6 +21,20 @@ from ...errors import TranslationError
_Raw = Mapping[str, object]
def explicit_stream_false(raw: _Raw) -> TranslationError | None:
"""The ONE explicit ``stream: false`` arm the azure / xai / compat_sdk
guards compose (critic-wave1a N2): those paths all serialize an
explicitly-sent ``false`` onto the wire while absent-vs-false is lost in
the IR. openai_compat itself deliberately does NOT run it -- the
gap-parity question is integrator scope (wave1a-port.md attack #2)."""
if raw.get("stream") is False:
return TranslationError.of_unsupported(
"explicit stream: false (this path keeps the key on the wire; "
"absent-vs-false is lost in the IR)"
)
return None
def unsupported_request_shapes(
raw: _Raw, *, name_fallback_user_only: bool = False
) -> TranslationError | None:
litellm/translation/providers/xai/guard.py
+4 -5
View File
@@ -27,6 +27,7 @@ from typing import cast
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from ...errors import TranslationError
from ..openai_compat.guard import explicit_stream_false
from ..openai_compat.guard import (
unsupported_request_shapes as openai_unsupported_request_shapes,
)
@@ -56,11 +57,9 @@ def unsupported_request_shapes(raw: _Raw) -> TranslationError | None:
"use_xai_oauth: v1 runs an interactive browser PKCE flow inside "
"validate_environment (llms/xai/oauth.py); v1 owns it"
)
if "stream" in raw and raw.get("stream") is False:
return TranslationError.of_unsupported(
"explicit stream: false (the xai httpx path keeps the key on the "
"wire; absent-vs-false is lost in the IR)"
)
stream_false = explicit_stream_false(raw)
if stream_false is not None:
return stream_false
reason = _nested_tool_strict_reason(raw)
if reason is not None:
return TranslationError.of_unsupported(reason)
tests/test_litellm/translation/_compat_sdk_corpus.py
+138 -120
View File
@@ -16,121 +16,139 @@ remap).
"""
import copy
from typing import Any, Dict
from collections.abc import Mapping
from types import MappingProxyType
from typing import Literal, NamedTuple
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_optional_params
# One spec per surviving wave-1a provider: the corpus model, which optional
# surfaces the provider serves (drives generated corpus/raise/fallback rows),
# and how max_completion_tokens behaves in v1 ("rename" -> max_tokens,
# "verbatim" -> passes through, "raise" -> outside the supported list).
SPECS: Dict[str, Dict[str, Any]] = {
"together_ai": {
"model": "Qwen/Qwen2.5-72B-Instruct-Turbo", # model-map: supports_function_calling
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "verbatim",
},
"cerebras": {
"model": "llama3.1-8b",
"tools": True,
"response_format": True,
"parallel_tool_calls": False,
"user": True,
"mct": "rename",
},
"nvidia_nim": {
"model": "meta/llama3-70b-instruct", # the default-list arm
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "rename",
},
"lm_studio": {
"model": "qwen2.5-7b-instruct-1m",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "verbatim",
},
"llamafile": {
"model": "LLaMA_CPP",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "verbatim",
},
"lambda_ai": {
"model": "llama3.1-70b-instruct-fp8",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "rename",
},
"nebius": {
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "rename",
},
"novita": {
"model": "meta-llama/llama-3.1-8b-instruct",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "verbatim",
},
"wandb": {
"model": "meta-llama/Llama-3.1-8B-Instruct",
"tools": True,
"response_format": True,
"parallel_tool_calls": True,
"user": False,
"mct": "rename",
},
"featherless_ai": {
"model": "featherless-ai/Qwerky-72B",
"tools": False,
"response_format": False,
"parallel_tool_calls": False,
"user": False,
"mct": "rename",
},
"nscale": {
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"tools": False,
"response_format": True,
"parallel_tool_calls": False,
"user": False,
"mct": "raise",
},
"hyperbolic": {
"model": "meta-llama/Meta-Llama-3-70B-Instruct",
"tools": True,
"response_format": True,
"parallel_tool_calls": False,
"user": True,
"mct": "raise",
},
"volcengine": {
"model": "doubao-pro-32k-241215",
"tools": True,
"response_format": False,
"parallel_tool_calls": False,
"user": False,
"mct": "rename",
},
}
_Case = dict[str, object]
class CompatSpec(NamedTuple):
"""One row per surviving wave-1a provider: the corpus model, which
optional surfaces the provider serves (drives generated
corpus/raise/fallback rows), and how max_completion_tokens behaves in
v1 ("rename" -> max_tokens, "verbatim" -> passes through, "raise" ->
outside the supported list)."""
model: str
tools: bool
response_format: bool
parallel_tool_calls: bool
user: bool
mct: Literal["rename", "verbatim", "raise"]
SPECS: Mapping[str, CompatSpec] = MappingProxyType(
{
"together_ai": CompatSpec(
model="Qwen/Qwen2.5-72B-Instruct-Turbo", # map: supports_function_calling
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="verbatim",
),
"cerebras": CompatSpec(
model="llama3.1-8b",
tools=True,
response_format=True,
parallel_tool_calls=False,
user=True,
mct="rename",
),
"nvidia_nim": CompatSpec(
model="meta/llama3-70b-instruct", # the default-list arm
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="rename",
),
"lm_studio": CompatSpec(
model="qwen2.5-7b-instruct-1m",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="verbatim",
),
"llamafile": CompatSpec(
model="LLaMA_CPP",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="verbatim",
),
"lambda_ai": CompatSpec(
model="llama3.1-70b-instruct-fp8",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="rename",
),
"nebius": CompatSpec(
model="meta-llama/Meta-Llama-3.1-70B-Instruct",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="rename",
),
"novita": CompatSpec(
model="meta-llama/llama-3.1-8b-instruct",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="verbatim",
),
"wandb": CompatSpec(
model="meta-llama/Llama-3.1-8B-Instruct",
tools=True,
response_format=True,
parallel_tool_calls=True,
user=False,
mct="rename",
),
"featherless_ai": CompatSpec(
model="featherless-ai/Qwerky-72B",
tools=False,
response_format=False,
parallel_tool_calls=False,
user=False,
mct="rename",
),
"nscale": CompatSpec(
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
tools=False,
response_format=True,
parallel_tool_calls=False,
user=False,
mct="raise",
),
"hyperbolic": CompatSpec(
model="meta-llama/Meta-Llama-3-70B-Instruct",
tools=True,
response_format=True,
parallel_tool_calls=False,
user=True,
mct="raise",
),
"volcengine": CompatSpec(
model="doubao-pro-32k-241215",
tools=True,
response_format=False,
parallel_tool_calls=False,
user=False,
mct="rename",
),
}
)
PROVIDERS = tuple(sorted(SPECS))
@@ -154,7 +172,7 @@ def provider_config(provider: str, model: str):
)
def run_v1_request_transform(provider: str, case: Dict[str, Any]) -> Dict[str, Any]:
def run_v1_request_transform(provider: str, case: _Case) -> dict:
request = copy.deepcopy(case)
model = request.pop("model")
messages = request.pop("messages")
@@ -176,13 +194,13 @@ def run_v1_request_transform(provider: str, case: Dict[str, Any]) -> Dict[str, A
)
def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]:
def corpus_for(provider: str) -> dict[str, _Case]:
"""The generated served corpus: every row here must be byte-identical
(normalized JSON) between v1-in-process and v2."""
spec = SPECS[provider]
model = spec["model"]
model = spec.model
user_msg = [{"role": "user", "content": "Hello, world"}]
cases: Dict[str, Dict[str, Any]] = {
cases: dict[str, _Case] = {
"text": {"model": model, "messages": user_msg},
"system_and_sampling": {
"model": model,
@@ -202,13 +220,13 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]:
"messages": user_msg,
},
}
if spec["mct"] in ("rename", "verbatim"):
if spec.mct in ("rename", "verbatim"):
cases["max_completion_tokens"] = {
"model": model,
"max_completion_tokens": 128,
"messages": user_msg,
}
if spec["tools"]:
if spec.tools:
cases["tools_auto"] = {
"model": model,
"tools": [WEATHER_TOOL],
@@ -243,7 +261,7 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]:
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
],
}
if spec["response_format"]:
if spec.response_format:
cases["response_format_json_object"] = {
"model": model,
"response_format": {"type": "json_object"},
@@ -266,13 +284,13 @@ def corpus_for(provider: str) -> Dict[str, Dict[str, Any]]:
},
"messages": user_msg,
}
if spec["parallel_tool_calls"]:
if spec.parallel_tool_calls:
cases["parallel_tool_calls_false"] = {
"model": model,
"tools": [WEATHER_TOOL],
"parallel_tool_calls": False,
"messages": [{"role": "user", "content": "Weather in Paris and Rome?"}],
}
if spec["user"]:
if spec.user:
cases["user_param"] = {"model": model, "user": "u-1", "messages": user_msg}
return cases
tests/test_litellm/translation/generate_differential_report.py
+1 -1
View File
@@ -328,7 +328,7 @@ def _compat_sdk_rows(lines: list) -> int:
]
for provider, name in resp._rows():
raw = resp._RESPONSES[name]
preset = f"{provider}/{corpus.SPECS[provider]['model']}"
preset = f"{provider}/{corpus.SPECS[provider].model}"
same = resp._norm(resp._v2_model_response(provider, raw, preset)) == resp._norm(
resp._v1_model_response(raw, preset)
)
tests/test_litellm/translation/test_differential_compat_sdk_request.py
+26 -14
View File
@@ -41,7 +41,7 @@ _TOOLS = [WEATHER_TOOL]
def _m(provider: str) -> str:
return SPECS[provider]["model"]
return SPECS[provider].model
# Rows where v1 RAISES UnsupportedParamsError (the supported-list gate); v2
@@ -51,19 +51,19 @@ def _m(provider: str) -> str:
V1_RAISES = {}
for _p in PROVIDERS:
_spec = SPECS[_p]
if _spec["mct"] == "raise":
if _spec.mct == "raise":
V1_RAISES[f"{_p}:max_completion_tokens"] = (
_p,
{"model": _m(_p), "max_completion_tokens": 128, "messages": _USER},
"max_completion_tokens",
)
if not _spec["tools"]:
if not _spec.tools:
V1_RAISES[f"{_p}:tools"] = (
_p,
{"model": _m(_p), "tools": _TOOLS, "messages": _USER},
"tools",
)
if not _spec["response_format"]:
if not _spec.response_format:
V1_RAISES[f"{_p}:response_format"] = (
_p,
{
@@ -73,7 +73,7 @@ for _p in PROVIDERS:
},
"response_format",
)
if not _spec["parallel_tool_calls"]:
if not _spec.parallel_tool_calls:
V1_RAISES[f"{_p}:parallel_tool_calls"] = (
_p,
{"model": _m(_p), "parallel_tool_calls": False, "messages": _USER},
@@ -231,7 +231,7 @@ for _p in PROVIDERS:
{"model": _m(_p), "seed": 42, "messages": _USER},
"seed",
)
if not SPECS[_p]["user"]:
if not SPECS[_p].user:
EXPECTED_FALLBACKS[f"{_p}:user_model_list_gate"] = (
_p,
{"model": _m(_p), "user": "u-1", "messages": _USER},
@@ -268,7 +268,7 @@ EXPECTED_FALLBACKS.update(
)
def _v2(provider: str, case: dict):
def _v2(provider: str, case: dict[str, object]):
return translate_chat_request(copy.deepcopy(case), provider, build_real_deps())
@@ -375,14 +375,14 @@ _MIRROR_KEYS = (
_RF_NAME_GATED = ("gpt-4", "gpt-3.5-turbo-16k")
def _base_family_allowed(model: str) -> frozenset:
def _base_family_allowed(model: str) -> frozenset[str]:
allowed = csp._BASE_LIST
if model in _RF_NAME_GATED:
return allowed - {"response_format"}
return allowed
def _v2_allowed(provider: str, model: str, deps) -> frozenset:
def _v2_allowed(provider: str, model: str, deps) -> frozenset[str]:
"""The per-model allowed set, re-derived from the SAME csp.ALLOWED table
serialization reads, with the three per-model narrowings the gates
apply (together capability fork, nvidia_nim static table, the base
@@ -450,7 +450,7 @@ def test_supported_list_mirrors_track_v1_at_head(provider: str) -> None:
)
for key in _MIRROR_KEYS:
assert (key in allowed) == (key in supported), (provider, model, key)
if SPECS[provider]["user"]:
if SPECS[provider].user:
assert "user" in supported, (provider, model)
if provider == "cerebras":
assert csp.supports_cerebras_reasoning(model, deps) == (
@@ -483,9 +483,7 @@ def test_capability_prefix_is_load_bearing() -> None:
) == litellm.supports_reasoning(model="qwen-3-32b", custom_llm_provider="cerebras")
@pytest.mark.parametrize(
"provider", [p for p in PROVIDERS if SPECS[p]["mct"] == "rename"]
)
@pytest.mark.parametrize("provider", [p for p in PROVIDERS if SPECS[p].mct == "rename"])
def test_mct_rename_matches_v1(provider: str) -> None:
"""The renamed key must be exactly what v1's map emits (max_tokens), so
the rename flag can never silently disagree with the v1 config."""
@@ -499,7 +497,7 @@ def test_mct_rename_matches_v1(provider: str) -> None:
@pytest.mark.parametrize(
"provider", [p for p in PROVIDERS if SPECS[p]["mct"] == "verbatim"]
"provider", [p for p in PROVIDERS if SPECS[p].mct == "verbatim"]
)
def test_mct_verbatim_matches_v1(provider: str) -> None:
case = {"model": _m(provider), "max_completion_tokens": 33, "messages": _USER}
@@ -511,6 +509,20 @@ def test_mct_verbatim_matches_v1(provider: str) -> None:
assert "max_tokens" not in result.ok
def test_top_k_fallback_reason_matches_v1_silent_drop() -> None:
"""v1 silently drops top_k WITHOUT drop_params (it is not an OpenAI
param, so _check_valid_arg never sees it) — the fallback reason must say
so instead of claiming an UnsupportedParamsError raise (verifier-wave1a
F6). Pinned in-process: v1 serves the request with top_k gone."""
case = {"model": _m("lambda_ai"), "top_k": 3, "messages": _USER}
result = _v2("lambda_ai", case)
assert result.is_error()
assert "silently drops it" in result.error.summary, result.error.summary
assert "UnsupportedParamsError" not in result.error.summary
v1 = run_v1_request_transform("lambda_ai", case)
assert "top_k" not in v1
def test_together_text_response_format_dropped_like_v1() -> None:
case = {
"model": _m("together_ai"),
tests/test_litellm/translation/test_differential_compat_sdk_response.py
+3 -3
View File
@@ -37,7 +37,7 @@ _RESPONSE_ROWS = (
def _request_for(provider: str) -> dict:
return {
"model": SPECS[provider]["model"],
"model": SPECS[provider].model,
"messages": [{"role": "user", "content": "hi"}],
}
@@ -74,7 +74,7 @@ def test_preset_reprefix_matches_v1(provider: str, name: str, frozen_ambient) ->
"""SDK-path preset: {provider}/{request model} in, {provider}/{wire
model} out, byte-identical dumps both sides."""
raw = _RESPONSES[name]
preset = f"{provider}/{SPECS[provider]['model']}"
preset = f"{provider}/{SPECS[provider].model}"
v1 = _v1_model_response(raw, preset)
v2 = _v2_model_response(provider, raw, preset)
assert _norm(v2) == _norm(v1)
@@ -87,7 +87,7 @@ def test_preset_survives_when_wire_model_missing(provider: str, frozen_ambient)
"""cdr's elif arm needs a non-None wire model; without one the preset
{provider}/{request model} survives verbatim on both sides."""
raw = {k: v for k, v in _RESPONSES["text"].items() if k != "model"}
preset = f"{provider}/{SPECS[provider]['model']}"
preset = f"{provider}/{SPECS[provider].model}"
v1 = _v1_model_response(raw, preset)
v2 = _v2_model_response(provider, raw, preset)
assert _norm(v2) == _norm(v1)