From ef8a25071709070387edaa64b18b08e7f02f367e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:56:08 +0000 Subject: [PATCH] test(translation): regenerate DIFFERENTIAL_REPORT.md with the compat_sdk family rows 0 divergent rows across all providers; every wave-1a fallback row is typed with the v1 path it names, and the baseten drop is recorded in the report itself --- .../translation/DIFFERENTIAL_REPORT.md | 392 +++++++++++++++++- .../generate_differential_report.py | 106 ++++- 2 files changed, 495 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md index acef118128..c8ea99e6f0 100644 --- a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md +++ b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md @@ -1,4 +1,4 @@ -# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai) +# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai + the wave-1a compat_sdk family) v1 and v2 run over the same corpus; every row must be IDENTICAL (or an explained FALLBACK that v1 serves) for a provider's flag to turn on. @@ -6,7 +6,7 @@ Bedrock and google rows additionally pin the characterization-corpus snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: `python -m tests.test_litellm.translation.generate_differential_report` -- commit: ae4643ba8a +- commit: b64b732772 ## anthropic: request bodies (v1 map_openai_params + transform_request vs v2) @@ -206,6 +206,394 @@ snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: - SEAM CONTRACT: usage_tail_include_usage (v1's chunk_parser injects a dummy choice so the wrapper swallows the tail and synthesizes the final usage chunk; v2 passes the wire choices=[] chunk through with the FOLDED usage for the streaming seam to synthesize from) - IDENTICAL: usage_withheld_on_content_chunks +## cerebras: request bodies (v1 get_optional_params('cerebras') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- IDENTICAL: user_param +- FALLBACK (v1 raises UnsupportedParamsError): cerebras:parallel_tool_calls (parallel_tool_calls) +- FALLBACK (v1 raises UnsupportedParamsError): cerebras:reasoning_effort_non_reasoning_model (reasoning_effort on non-reasoning cerebras model) +- FALLBACK (v1 serves it): cerebras:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): cerebras:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): cerebras:message_name_field (message name field) +- FALLBACK (v1 serves it): cerebras:seed_outside_ir (seed) +- FALLBACK (v1 serves it): cerebras:string_form_stop (string-form stop) + +## featherless_ai: request bodies (v1 get_optional_params('featherless_ai') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- FALLBACK (v1 raises UnsupportedParamsError): featherless_ai:parallel_tool_calls (parallel_tool_calls) +- FALLBACK (v1 raises UnsupportedParamsError): featherless_ai:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): featherless_ai:response_format (response_format) +- FALLBACK (v1 raises UnsupportedParamsError): featherless_ai:tools (tools) +- FALLBACK (v1 serves it): featherless_ai:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): featherless_ai:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): featherless_ai:message_name_field (message name field) +- FALLBACK (v1 serves it): featherless_ai:seed_outside_ir (seed) +- FALLBACK (v1 serves it): featherless_ai:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): featherless_ai:user_model_list_gate (user) + +## hyperbolic: request bodies (v1 get_optional_params('hyperbolic') + transform_request vs v2 compat_sdk) + +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- IDENTICAL: user_param +- FALLBACK (v1 raises UnsupportedParamsError): hyperbolic:max_completion_tokens (max_completion_tokens) +- FALLBACK (v1 raises UnsupportedParamsError): hyperbolic:parallel_tool_calls (parallel_tool_calls) +- FALLBACK (v1 raises UnsupportedParamsError): hyperbolic:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): hyperbolic:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): hyperbolic:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): hyperbolic:message_name_field (message name field) +- FALLBACK (v1 serves it): hyperbolic:seed_outside_ir (seed) +- FALLBACK (v1 serves it): hyperbolic:string_form_stop (string-form stop) + +## lambda_ai: request bodies (v1 get_optional_params('lambda_ai') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): lambda_ai:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): lambda_ai:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): lambda_ai:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): lambda_ai:message_name_field (message name field) +- FALLBACK (v1 serves it): lambda_ai:seed_outside_ir (seed) +- FALLBACK (v1 serves it): lambda_ai:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): lambda_ai:user_model_list_gate (user) + +## llamafile: request bodies (v1 get_optional_params('llamafile') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): llamafile:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): llamafile:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): llamafile:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): llamafile:message_name_field (message name field) +- FALLBACK (v1 serves it): llamafile:seed_outside_ir (seed) +- FALLBACK (v1 serves it): llamafile:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): llamafile:user_model_list_gate (user) + +## lm_studio: request bodies (v1 get_optional_params('lm_studio') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): lm_studio:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): lm_studio:response_format_on_gpt4_name (outside v1's supported set) +- FALLBACK (v1 serves it): lm_studio:bare_schema_response_format (response_format) +- FALLBACK (v1 serves it): lm_studio:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): lm_studio:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): lm_studio:message_name_field (message name field) +- FALLBACK (v1 serves it): lm_studio:seed_outside_ir (seed) +- FALLBACK (v1 serves it): lm_studio:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): lm_studio:user_model_list_gate (user) + +## nebius: request bodies (v1 get_optional_params('nebius') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): nebius:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): nebius:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): nebius:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): nebius:message_name_field (message name field) +- FALLBACK (v1 serves it): nebius:seed_outside_ir (seed) +- FALLBACK (v1 serves it): nebius:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): nebius:user_model_list_gate (user) + +## novita: request bodies (v1 get_optional_params('novita') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): novita:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): novita:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): novita:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): novita:message_name_field (message name field) +- FALLBACK (v1 serves it): novita:seed_outside_ir (seed) +- FALLBACK (v1 serves it): novita:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): novita:user_model_list_gate (user) + +## nscale: request bodies (v1 get_optional_params('nscale') + transform_request vs v2 compat_sdk) + +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- FALLBACK (v1 raises UnsupportedParamsError): nscale:max_completion_tokens (max_completion_tokens) +- FALLBACK (v1 raises UnsupportedParamsError): nscale:parallel_tool_calls (parallel_tool_calls) +- FALLBACK (v1 raises UnsupportedParamsError): nscale:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): nscale:tools (tools) +- FALLBACK (v1 serves it): nscale:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): nscale:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): nscale:message_name_field (message name field) +- FALLBACK (v1 serves it): nscale:seed_outside_ir (seed) +- FALLBACK (v1 serves it): nscale:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): nscale:user_model_list_gate (user) + +## nvidia_nim: request bodies (v1 get_optional_params('nvidia_nim') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): nvidia_nim:max_completion_tokens_on_gemma (max_completion_tokens) +- FALLBACK (v1 raises UnsupportedParamsError): nvidia_nim:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): nvidia_nim:stop_on_nemotron_instruct (stop) +- FALLBACK (v1 raises UnsupportedParamsError): nvidia_nim:temperature_on_nemotron_reward (temperature) +- FALLBACK (v1 raises UnsupportedParamsError): nvidia_nim:tools_on_gemma (tools) +- FALLBACK (v1 serves it): nvidia_nim:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): nvidia_nim:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): nvidia_nim:message_name_field (message name field) +- FALLBACK (v1 serves it): nvidia_nim:seed_outside_ir (seed) +- FALLBACK (v1 serves it): nvidia_nim:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): nvidia_nim:user_model_list_gate (user) + +## together_ai: request bodies (v1 get_optional_params('together_ai') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): together_ai:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): together_ai:response_format_on_non_fc_model (response_format) +- FALLBACK (v1 raises UnsupportedParamsError): together_ai:tools_on_non_fc_model (tools) +- FALLBACK (v1 serves it): together_ai:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): together_ai:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): together_ai:message_name_field (message name field) +- FALLBACK (v1 serves it): together_ai:seed_outside_ir (seed) +- FALLBACK (v1 serves it): together_ai:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): together_ai:user_model_list_gate (user) + +## volcengine: request bodies (v1 get_optional_params('volcengine') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): volcengine:parallel_tool_calls (parallel_tool_calls) +- FALLBACK (v1 raises UnsupportedParamsError): volcengine:reasoning_effort (reasoning_effort) +- FALLBACK (v1 raises UnsupportedParamsError): volcengine:response_format (response_format) +- FALLBACK (v1 serves it): volcengine:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): volcengine:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): volcengine:message_name_field (message name field) +- FALLBACK (v1 serves it): volcengine:seed_outside_ir (seed) +- FALLBACK (v1 serves it): volcengine:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): volcengine:thinking_extra_body_packing (extra_body) +- FALLBACK (v1 serves it): volcengine:user_model_list_gate (user) + +## wandb: request bodies (v1 get_optional_params('wandb') + transform_request vs v2 compat_sdk) + +- IDENTICAL: max_completion_tokens +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: stop_list +- IDENTICAL: stream_true +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_compact_roundtrip +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- FALLBACK (v1 raises UnsupportedParamsError): wandb:reasoning_effort (reasoning_effort) +- FALLBACK (v1 serves it): wandb:both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): wandb:explicit_stream_false (explicit stream: false) +- FALLBACK (v1 serves it): wandb:message_name_field (message name field) +- FALLBACK (v1 serves it): wandb:seed_outside_ir (seed) +- FALLBACK (v1 serves it): wandb:string_form_stop (string-form stop) +- FALLBACK (v1 serves it): wandb:user_model_list_gate (user) + +## compat_sdk family: responses (v1 convert_to_model_response_object with the SDK-path {provider}/{model} preset vs v2 + seam re-prefix arm) + +- IDENTICAL: cerebras cached_and_reasoning_usage_details +- IDENTICAL: cerebras text +- IDENTICAL: cerebras tool_calls_rewrites_stop +- IDENTICAL: featherless_ai cached_and_reasoning_usage_details +- IDENTICAL: featherless_ai text +- IDENTICAL: featherless_ai tool_calls_rewrites_stop +- IDENTICAL: hyperbolic cached_and_reasoning_usage_details +- IDENTICAL: hyperbolic text +- IDENTICAL: hyperbolic tool_calls_rewrites_stop +- IDENTICAL: lambda_ai cached_and_reasoning_usage_details +- IDENTICAL: lambda_ai text +- IDENTICAL: lambda_ai tool_calls_rewrites_stop +- IDENTICAL: llamafile cached_and_reasoning_usage_details +- IDENTICAL: llamafile text +- IDENTICAL: llamafile tool_calls_rewrites_stop +- IDENTICAL: lm_studio cached_and_reasoning_usage_details +- IDENTICAL: lm_studio text +- IDENTICAL: lm_studio tool_calls_rewrites_stop +- IDENTICAL: nebius cached_and_reasoning_usage_details +- IDENTICAL: nebius text +- IDENTICAL: nebius tool_calls_rewrites_stop +- IDENTICAL: novita cached_and_reasoning_usage_details +- IDENTICAL: novita text +- IDENTICAL: novita tool_calls_rewrites_stop +- IDENTICAL: nscale cached_and_reasoning_usage_details +- IDENTICAL: nscale text +- IDENTICAL: nscale tool_calls_rewrites_stop +- IDENTICAL: nvidia_nim cached_and_reasoning_usage_details +- IDENTICAL: nvidia_nim text +- IDENTICAL: nvidia_nim tool_calls_rewrites_stop +- IDENTICAL: together_ai cached_and_reasoning_usage_details +- IDENTICAL: together_ai text +- IDENTICAL: together_ai tool_calls_rewrites_stop +- IDENTICAL: volcengine cached_and_reasoning_usage_details +- IDENTICAL: volcengine text +- IDENTICAL: volcengine tool_calls_rewrites_stop +- IDENTICAL: wandb cached_and_reasoning_usage_details +- IDENTICAL: wandb text +- IDENTICAL: wandb tool_calls_rewrites_stop + +## compat_sdk family: streams (v1 CustomStreamWrapper(provider) over SDK chunks vs v2 openai dialect) + +- IDENTICAL: cerebras empty_keepalive_swallowed +- IDENTICAL: cerebras text +- IDENTICAL: cerebras tools +- IDENTICAL: featherless_ai empty_keepalive_swallowed +- IDENTICAL: featherless_ai text +- IDENTICAL: featherless_ai tools +- IDENTICAL: hyperbolic empty_keepalive_swallowed +- IDENTICAL: hyperbolic text +- IDENTICAL: hyperbolic tools +- IDENTICAL: lambda_ai empty_keepalive_swallowed +- IDENTICAL: lambda_ai text +- IDENTICAL: lambda_ai tools +- IDENTICAL: llamafile empty_keepalive_swallowed +- IDENTICAL: llamafile text +- IDENTICAL: llamafile tools +- IDENTICAL: lm_studio empty_keepalive_swallowed +- IDENTICAL: lm_studio text +- IDENTICAL: lm_studio tools +- IDENTICAL: nebius empty_keepalive_swallowed +- IDENTICAL: nebius text +- IDENTICAL: nebius tools +- IDENTICAL: novita empty_keepalive_swallowed +- IDENTICAL: novita text +- IDENTICAL: novita tools +- IDENTICAL: nscale empty_keepalive_swallowed +- IDENTICAL: nscale text +- IDENTICAL: nscale tools +- IDENTICAL: nvidia_nim empty_keepalive_swallowed +- IDENTICAL: nvidia_nim text +- IDENTICAL: nvidia_nim tools +- IDENTICAL: together_ai empty_keepalive_swallowed +- IDENTICAL: together_ai text +- IDENTICAL: together_ai tools +- IDENTICAL: volcengine empty_keepalive_swallowed +- IDENTICAL: volcengine text +- IDENTICAL: volcengine tools +- IDENTICAL: wandb empty_keepalive_swallowed +- IDENTICAL: wandb text +- IDENTICAL: wandb tools +- SEAM CONTRACT: cerebras usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: featherless_ai usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: hyperbolic usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: lambda_ai usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: llamafile usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: lm_studio usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: nebius usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: novita usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: nscale usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: nvidia_nim usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: together_ai usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: volcengine usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) +- SEAM CONTRACT: wandb usage tail (v2 passes the wire choices=[] usage chunk through; the streaming seam owns v1's synthesized final chunk) + +- DROPPED FROM WAVE 1A: baseten (streams ride the dedicated legacy handle_baseten_chunk wrapper branch, not the openai dialect; unregistered, typed v1 fallback; canary test_baseten_drop_canary pins the evidence) + ## azure: request bodies (v1 api-version-aware map_openai_params + transform_request vs v2) - IDENTICAL: deployment_with_base_model diff --git a/tests/test_litellm/translation/generate_differential_report.py b/tests/test_litellm/translation/generate_differential_report.py index 275296fa0f..14931166a8 100644 --- a/tests/test_litellm/translation/generate_differential_report.py +++ b/tests/test_litellm/translation/generate_differential_report.py @@ -261,6 +261,109 @@ def _xai_rows(lines: list) -> int: return failures +def _compat_sdk_rows(lines: list) -> int: + from litellm.exceptions import UnsupportedParamsError + + from . import _compat_sdk_corpus as corpus + from . import test_differential_compat_sdk_request as req + from . import test_differential_compat_sdk_response as resp + from . import test_differential_compat_sdk_stream as stream + + failures = 0 + for provider in corpus.PROVIDERS: + lines += [ + "", + f"## {provider}: request bodies (v1 get_optional_params('{provider}')" + " + transform_request vs v2 compat_sdk)", + "", + ] + for name in sorted(corpus.corpus_for(provider)): + case = corpus.corpus_for(provider)[name] + result = req._v2(provider, case) + same = result.is_ok() and req._norm(result.ok) == req._norm( + corpus.run_v1_request_transform(provider, case) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + for name in sorted(k for k in req.V1_RAISES if k.startswith(f"{provider}:")): + p, case, reason = req.V1_RAISES[name] + result = req._v2(p, case) + try: + corpus.run_v1_request_transform(p, case) + raised = False + except UnsupportedParamsError: + raised = True + ok = result.is_error() and reason in result.error.summary and raised + failures += 0 if ok else 1 + label = "FALLBACK (v1 raises UnsupportedParamsError)" if ok else "DIVERGENT" + lines.append(f"- {label}: {name} ({reason})") + for name in sorted( + k for k in req.EXPECTED_FALLBACKS if k.startswith(f"{provider}:") + ): + p, case, reason = req.EXPECTED_FALLBACKS[name] + result = req._v2(p, case) + ok = result.is_error() and reason in result.error.summary + failures += 0 if ok else 1 + label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT" + lines.append(f"- {label}: {name} ({reason})") + lines += [ + "", + "## compat_sdk family: responses (v1 convert_to_model_response_object" + " with the SDK-path {provider}/{model} preset vs v2 + seam re-prefix arm)", + "", + ] + for provider, name in resp._rows(): + raw = resp._RESPONSES[name] + preset = f"{provider}/{corpus.SPECS[provider]['model']}" + same = resp._norm(resp._v2_model_response(provider, raw, preset)) == resp._norm( + resp._v1_model_response(raw, preset) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {provider} {name}") + lines += [ + "", + "## compat_sdk family: streams (v1 CustomStreamWrapper(provider) over" + " SDK chunks vs v2 openai dialect)", + "", + ] + for provider, name in stream._rows(): + events = stream.STREAMS[name] + same = stream._norm(stream._v2_chunks(events)) == stream._norm( + stream._v1_chunks(provider, events) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {provider} {name}") + for provider in corpus.PROVIDERS: + v1 = stream._v1_chunks( + provider, stream.USAGE_STREAM, stream_options={"include_usage": True} + ) + v2 = stream._v2_chunks(stream.USAGE_STREAM) + tail_ok = ( + len(v1) == len(v2) + and stream._norm(v2[:-1]) == stream._norm(v1[: len(v2) - 1]) + and v2[-1]["choices"] == [] + and all( + v1[-1]["usage"][k] == v2[-1]["usage"][k] + for k in ("prompt_tokens", "completion_tokens", "total_tokens") + ) + ) + failures += 0 if tail_ok else 1 + lines.append( + ("- SEAM CONTRACT: " if tail_ok else "- DIVERGENT: ") + + f"{provider} usage tail (v2 passes the wire choices=[] usage" + " chunk through; the streaming seam owns v1's synthesized final" + " chunk)" + ) + lines += [ + "", + "- DROPPED FROM WAVE 1A: baseten (streams ride the dedicated legacy" + " handle_baseten_chunk wrapper branch, not the openai dialect;" + " unregistered, typed v1 fallback; canary" + " test_baseten_drop_canary pins the evidence)", + ] + return failures + + def _azure_rows(lines: list) -> int: import os @@ -732,7 +835,7 @@ def main() -> None: _stub_vertex_token() lines = [ - "# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai)", + "# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai + the wave-1a compat_sdk family)", "", "v1 and v2 run over the same corpus; every row must be IDENTICAL (or an", "explained FALLBACK that v1 serves) for a provider's flag to turn on.", @@ -746,6 +849,7 @@ def main() -> None: failures = _anthropic_rows(lines) failures += _openai_rows(lines) failures += _xai_rows(lines) + failures += _compat_sdk_rows(lines) failures += _azure_rows(lines) failures += _azure_ai_rows(lines) failures += _bedrock_request_rows(lines)