test(translation): openai_compat registry rows, raw-guard table, and two-sided differential gates

Append-only registry rows (serializer/parser/dialect openai) plus a
_RAW_GUARDS table the pipeline consults before parse for same-family
providers. Request gate: 17 identical rows against v1 in-process
(map_openai_params + transform_request) and 17 asserted typed
fallbacks. Response gate: 5 identical rows against
convert_to_model_response_object (the live SDK-path normalizer) plus
loud-error rows for unreachable shapes. Stream gate: byte-identical
replays through the real CustomStreamWrapper over SDK chunks, an
SSE-line decode equivalence check, and the pinned usage-tail seam
contract. DIFFERENTIAL_REPORT.md regenerated: 0 divergent rows.
This commit is contained in:
mateo-berri
2026-06-12 01:55:01 +00:00
parent c1fee272a0
commit 91a84b7edf
9 changed files with 1199 additions and 23 deletions
+47 -19
View File
@@ -55,13 +55,27 @@ translation/
│ │ ├── response.py # converse JSON -> IR (properties-unwrap json rewrite)
│ │ └── stream.py # PARSED converse events -> IR events (pinned at the
│ │ # parsed-event seam; AWS framing is botocore's)
── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY:
├── serialize.py # pop model/stream, inject anthropic_version, spoof
│ # model for response_format (v1's json-tool forcing)
├── response.py # re-export of anthropic parse_response (invoke
│ # response body IS anthropic wire format)
└── stream.py # re-export of anthropic parse_event (invoke stream
# = anthropic events over AWS framing)
── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY:
├── serialize.py # pop model/stream, inject anthropic_version, spoof
│ # model for response_format (v1's json-tool forcing)
├── response.py # re-export of anthropic parse_response (invoke
│ # response body IS anthropic wire format)
└── stream.py # re-export of anthropic parse_event (invoke stream
# = anthropic events over AWS framing)
│ └── openai_compat/ # the same-family hub serializer (GPT first consumer)
│ ├── guard.py # raw-shape fidelity guard run BEFORE parse: shapes
│ │ # the IR cannot round-trip losslessly (string stop,
│ │ # message name, image detail, max-tokens key split,
│ │ # tool-arg spacing) fall back to v1 as typed errors
│ ├── serialize.py # v1's five-touch passthrough body assembly
│ ├── messages.py # IR -> openai wire messages (inverse of inbound)
│ ├── params.py # o-series/gpt-5 family gates (fail closed until
│ │ # their param families are ported), user gate
│ ├── response.py # mirrors convert_to_model_response_object (the LIVE
│ │ # normalizer; transform_response is dead on the SDK
│ │ # path); rides the outbound body on ChatResponse.wire
│ └── stream.py # SSE chunk -> wire_chunk events normalized to the
│ # SDK-dump shape; the openai chunk dialect folds them
└── engine/
├── pipeline.py # prepare (pure, drives the fallback decision) -> send;
│ # per-provider serializer/parser/dialect tables; the
@@ -178,22 +192,36 @@ A behavior change ships as its own snapshot-diffed PR, never inside a port.
## Current scope
OpenAI-chat-in to three providers out — `anthropic`, `bedrock_converse`,
`bedrock_invoke` — request, response, and stream translation,
differential-green (anthropic: 46-shape corpus + responses + stream replays;
bedrock: the characterization corpus per route + quirk corpus), fail-closed
everywhere else, with non-streaming flag-gated seams live in `completion()`.
OpenAI-chat-in to four providers out — `anthropic`, `bedrock_converse`,
`bedrock_invoke`, `openai_compat` — request, response, and stream
translation, differential-green (anthropic: 46-shape corpus + responses +
stream replays; bedrock: the characterization corpus per route + quirk
corpus; openai: 17-shape request corpus + 17 typed-fallback rows + response
and SDK-chunk stream replays), fail-closed everywhere else, with
non-streaming flag-gated seams live in `completion()` for the anthropic and
bedrock routes (the openai seam fork is integrator scope and NOT wired).
Deliberate bedrock fallback surfaces (each names the v1 path): non-Claude
bedrock models, native structured outputs (outputConfig), adaptive-effort
output_config/beta, response_format+stream (fake_stream), response_format
with thinking on invoke (the model spoof crossing), tool history without
tools (modify_params dummy tool), empty user text on converse
(string-vs-list ambiguity), provisioned `model_id`, `guardrailConfig`, and
`<thinking>`-tagged text in converse responses. Not yet here, each its own
follow-up: streaming seams live; the other inbound schemas
(`anthropic_messages`, `google_genai`, `responses`, `completions`); the other
providers (vertex, azure, `openai_compat`); the same-family fast path
(waits on the opaque-body relay). To add a provider: write
`<thinking>`-tagged text in converse responses. Deliberate openai fallback
surfaces: o-series and gpt-5 model families (their param-rewrite configs are
unported), every raw shape the IR cannot round-trip byte-identically (the
guard's list: string stop, both max-tokens keys, message `name`, image
`detail`/`format`, consecutive same-role turns, single-text content lists,
empty tools/stop lists, non-canonical tool-argument JSON spacing, null/
non-function tool_calls), the `user` param (model-list gated in v1),
`response_format` on gpt-4/gpt-3.5-turbo-16k, `stream_options`, file blocks
(v1 downloads http pdf file_ids in-transform), and `http://` image URLs. On
streams, the trailing `choices: []` usage chunk passes through verbatim and
the wrapper's synthesized final usage chunk stays a seam/envelope concern.
Not yet here, each its own follow-up: streaming seams live; the other
inbound schemas (`anthropic_messages`, `google_genai`, `responses`,
`completions`); the other providers (vertex, azure); the same-family fast
path (waits on the opaque-body relay). To add a provider: write
`providers/<name>/`, register it in `engine/pipeline._SERIALIZERS` /
`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS`, add a differential corpus, keep
the flag off until differential-green.
`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS` (plus `_RAW_GUARDS` when the
inbound schema is the provider's own family), add a differential corpus,
keep the flag off until differential-green.
+34
View File
@@ -39,6 +39,13 @@ from ..providers.bedrock_invoke import parse_response as bedrock_invoke_parse_re
from ..providers.bedrock_invoke import (
serialize_request as bedrock_invoke_serialize_request,
)
from ..providers.openai_compat import parse_response as openai_compat_parse_response
from ..providers.openai_compat import (
serialize_request as openai_compat_serialize_request,
)
from ..providers.openai_compat import (
unsupported_request_shapes as openai_compat_unsupported_request_shapes,
)
from .http import Endpoint, ExecuteError, HttpPort, ProviderHttpError
_Serializer = Callable[[ChatRequest, TranslationDeps], Result[Body, TranslationError]]
@@ -51,6 +58,7 @@ _SERIALIZERS: Mapping[Provider, _Serializer] = MappingProxyType(
"anthropic": serialize_request,
"bedrock_converse": bedrock_converse_serialize_request,
"bedrock_invoke": bedrock_invoke_serialize_request,
"openai_compat": openai_compat_serialize_request,
}
)
@@ -59,6 +67,7 @@ _RESPONSE_PARSERS: Mapping[Provider, _ResponseParser] = MappingProxyType(
"anthropic": parse_response,
"bedrock_converse": bedrock_converse_parse_response,
"bedrock_invoke": bedrock_invoke_parse_response,
"openai_compat": openai_compat_parse_response,
}
)
@@ -67,9 +76,28 @@ _RESPONSE_DIALECTS: Mapping[Provider, ResponseDialect] = MappingProxyType(
"anthropic": "anthropic",
"bedrock_converse": "bedrock_converse",
"bedrock_invoke": "anthropic", # invoke delegates to the anthropic transform
"openai_compat": "openai", # same-family: the wire-derived body
}
)
_RawGuard = Callable[[Mapping[str, object]], TranslationError | None]
_RAW_GUARDS: Mapping[Provider, _RawGuard] = MappingProxyType(
# Same-family providers run a raw-shape fidelity guard BEFORE parse: the
# inbound parse normalizes wire forms v1 forwards verbatim, so shapes it
# cannot round-trip losslessly fall back to v1 as typed errors.
{
"openai_compat": openai_compat_unsupported_request_shapes,
}
)
def _raw_guard_error(
raw: Mapping[str, object], provider: Provider
) -> TranslationError | None:
guard = _RAW_GUARDS.get(provider)
return guard(raw) if guard is not None else None
def response_dialect(provider: Provider) -> ResponseDialect:
return _RESPONSE_DIALECTS.get(provider, "anthropic")
@@ -85,6 +113,9 @@ def translate_chat_request(
f"provider {provider!r} has no v2 chat serializer yet"
)
)
guard_error = _raw_guard_error(raw, provider)
if guard_error is not None:
return Error(guard_error)
return parse_request(raw).bind(lambda request: serializer(request, deps))
@@ -127,6 +158,9 @@ def prepare_chat_request(
f"provider {provider!r} is not fully ported to v2 yet"
)
)
guard_error = _raw_guard_error(raw, provider)
if guard_error is not None:
return Error(guard_error)
match parse_request(raw):
case Result(tag="ok", ok=request):
pass
@@ -0,0 +1,12 @@
from .guard import unsupported_request_shapes
from .response import parse_response
from .serialize import serialize_request
from .stream import parse_event, parse_line
__all__ = (
"parse_event",
"parse_line",
"parse_response",
"serialize_request",
"unsupported_request_shapes",
)
@@ -16,9 +16,10 @@ from __future__ import annotations
from expression import Error, Ok, Option, Result
from expression.collections import Block
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from typing_extensions import assert_never
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from ...deps import TranslationDeps
from ...errors import TranslationError
from ...ir import Body, ChatRequest, PlainJson, ResponseFormat, ToolChoice, ToolDef
@@ -1,4 +1,4 @@
# Translation v2 differential report (anthropic + bedrock)
# Translation v2 differential report (anthropic + bedrock + openai)
v1 and v2 run over the same corpus; every row must be IDENTICAL (or an
explained FALLBACK that v1 serves) for a provider's flag to turn on.
@@ -6,7 +6,7 @@ Bedrock rows additionally pin the characterization-corpus snapshot, so
each row proves snapshot == v1-at-HEAD == v2. Regenerate with:
`python -m tests.test_litellm.translation.generate_differential_report`
- commit: a10779c756
- commit: c1fee272a0
## anthropic: request bodies (v1 map_openai_params + transform_request vs v2)
@@ -76,6 +76,59 @@ each row proves snapshot == v1-at-HEAD == v2. Regenerate with:
- IDENTICAL: thinking
- IDENTICAL: tools
## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2)
- IDENTICAL: cache_control_stripped_everywhere
- IDENTICAL: image_base64
- IDENTICAL: image_url_string_to_object
- IDENTICAL: max_completion_tokens
- IDENTICAL: multiturn_stop_list_stream
- IDENTICAL: parallel_tool_calls_false
- IDENTICAL: response_format_json_object
- IDENTICAL: response_format_json_schema_strict
- IDENTICAL: system_and_sampling
- IDENTICAL: temperature_int_stays_int
- IDENTICAL: text
- IDENTICAL: tool_call_roundtrip
- IDENTICAL: tool_choice_none
- IDENTICAL: tool_choice_required
- IDENTICAL: tool_choice_specific
- IDENTICAL: tools_auto
- IDENTICAL: tools_strict
- FALLBACK (v1 serves it): both_max_tokens_keys (both max_tokens and max_completion_tokens)
- FALLBACK (v1 serves it): consecutive_user_messages (consecutive user messages)
- FALLBACK (v1 serves it): empty_tools_list (empty tools list)
- FALLBACK (v1 serves it): gpt5_model (OpenAIGPT5Config)
- FALLBACK (v1 serves it): http_pdf_file_id (messages)
- FALLBACK (v1 serves it): image_detail_key (image_url detail/format)
- FALLBACK (v1 serves it): legacy_function_call (function_call)
- FALLBACK (v1 serves it): message_name_field (message name field)
- FALLBACK (v1 serves it): o_series_model (OpenAIOSeriesConfig)
- FALLBACK (v1 serves it): reasoning_effort_plain_gpt (reasoning_effort)
- FALLBACK (v1 serves it): response_format_on_gpt4 (outside v1's supported set)
- FALLBACK (v1 serves it): single_text_content_list (single-text content list)
- FALLBACK (v1 serves it): stop_string_form (string-form stop)
- FALLBACK (v1 serves it): stream_options_unsupported (stream_options)
- FALLBACK (v1 serves it): tool_call_compact_arguments (non-canonical JSON spacing)
- FALLBACK (v1 serves it): top_k_not_openai (top_k)
- FALLBACK (v1 serves it): user_param_model_list_gate (open_ai_chat_completion_models)
## openai_compat: responses (v1 convert_to_model_response_object vs v2)
- IDENTICAL: cached_and_reasoning_usage_details
- IDENTICAL: reasoning_content_key
- IDENTICAL: text
- IDENTICAL: think_tag_extraction
- IDENTICAL: tool_calls_rewrites_stop
## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold)
- IDENTICAL: empty_keepalive_swallowed
- IDENTICAL: text
- IDENTICAL: text_no_leading_role
- IDENTICAL: tools
- SEAM CONTRACT: usage tail (v2 passes the wire choices=[] usage chunk through; v1's wrapper synthesizes its final usage chunk from it, which is the streaming seam's envelope to reproduce)
## bedrock_converse: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON)
- IDENTICAL: cache_control_messages
@@ -63,6 +63,77 @@ def _anthropic_rows(lines: list) -> int:
return failures
def _openai_rows(lines: list) -> int:
from litellm.translation import translate_chat_request
from . import test_differential_openai_request as req
from . import test_differential_openai_response as resp
from . import test_differential_openai_stream as stream
from .conftest import build_real_deps
failures = 0
lines += [
"",
"## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2)",
"",
]
for name in sorted(req.CORPUS):
result = req._v2_body(req.CORPUS[name])
same = result.is_ok() and req._norm(result.ok) == req._norm(
req._v1_body(req.CORPUS[name])
)
failures += 0 if same else 1
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
for name in sorted(req.EXPECTED_FALLBACKS):
case, reason = req.EXPECTED_FALLBACKS[name]
result = translate_chat_request(dict(case), "openai_compat", build_real_deps())
ok = result.is_error() and reason in result.error.summary
failures += 0 if ok else 1
label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT"
lines.append(f"- {label}: {name} ({reason})")
lines += [
"",
"## openai_compat: responses (v1 convert_to_model_response_object vs v2)",
"",
]
for name in sorted(resp._RESPONSES):
same = resp._norm(resp._v2_model_response(resp._RESPONSES[name])) == resp._norm(
resp._v1_model_response(resp._RESPONSES[name])
)
failures += 0 if same else 1
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
lines += [
"",
"## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold)",
"",
]
for name in sorted(stream.STREAMS):
same = stream._norm(stream._v2_chunks(stream.STREAMS[name])) == stream._norm(
stream._v1_chunks(stream.STREAMS[name])
)
failures += 0 if same else 1
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
v1 = stream._v1_chunks(stream.USAGE_STREAM, stream_options={"include_usage": True})
v2 = stream._v2_chunks(stream.USAGE_STREAM)
tail_ok = (
len(v1) == len(v2)
and stream._norm(v2[:-1]) == stream._norm(v1[: len(v2) - 1])
and v2[-1]["choices"] == []
and all(
v1[-1]["usage"][k] == v2[-1]["usage"][k]
for k in ("prompt_tokens", "completion_tokens", "total_tokens")
)
)
failures += 0 if tail_ok else 1
lines.append(
("- SEAM CONTRACT: " if tail_ok else "- DIVERGENT: ")
+ "usage tail (v2 passes the wire choices=[] usage chunk through; v1's"
" wrapper synthesizes its final usage chunk from it, which is the"
" streaming seam's envelope to reproduce)"
)
return failures
def _bedrock_request_rows(lines: list) -> int:
from litellm.translation import translate_chat_request
@@ -195,7 +266,7 @@ def main() -> None:
_freeze_ambient()
lines = [
"# Translation v2 differential report (anthropic + bedrock)",
"# Translation v2 differential report (anthropic + bedrock + openai)",
"",
"v1 and v2 run over the same corpus; every row must be IDENTICAL (or an",
"explained FALLBACK that v1 serves) for a provider's flag to turn on.",
@@ -207,6 +278,7 @@ def main() -> None:
"",
]
failures = _anthropic_rows(lines)
failures += _openai_rows(lines)
failures += _bedrock_request_rows(lines)
failures += _bedrock_response_rows(lines)
failures += _bedrock_stream_rows(lines)
@@ -0,0 +1,432 @@
"""Differential parity: v2 IR translation vs the v1 OpenAIGPTConfig chain.
For each OpenAI chat request in the corpus, the v1 body is produced exactly as
``litellm.completion`` would for provider "openai" (``map_openai_params`` then
``transform_request``) and compared, as normalized JSON, to the v2 body from
``translate_chat_request``. The corpus is the covered surface: the
openai_compat v2 flag only turns on for the shapes pinned here. v1 is a
near-passthrough (five touches), so byte parity is only possible for shapes
the inbound parse round-trips losslessly; everything else must be a TYPED
fallback (the raw guard / parse / serializer reasons asserted below), never a
silent divergence.
"""
import copy
import json
import pytest
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.translation import translate_chat_request
from .conftest import build_real_deps
MODEL = "gpt-4o"
_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
_STRICT_TOOL = {
"type": "function",
"function": {
"name": "report",
"parameters": {
"type": "object",
"properties": {"body": {"type": "string"}},
"required": ["body"],
"additionalProperties": False,
},
"strict": True,
},
}
def _assistant_tool_call(call_id, city, name="get_weather"):
return {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": call_id,
"type": "function",
# json.dumps default spacing: the only argument form the IR
# re-dumps byte-identically (the guard rejects the rest)
"function": {"name": name, "arguments": json.dumps({"city": city})},
}
],
}
CORPUS = {
"text": {
"model": MODEL,
"messages": [{"role": "user", "content": "Hello, world"}],
},
"system_and_sampling": {
"model": MODEL,
"max_tokens": 50,
"temperature": 0.5,
"top_p": 0.9,
"messages": [
{"role": "system", "content": "You are helpful"},
{"role": "user", "content": "Hi"},
],
},
"multiturn_stop_list_stream": {
"model": MODEL,
"max_tokens": 64,
"stop": ["END", "STOP"],
"stream": True,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
{"role": "user", "content": "How are you?"},
],
},
"max_completion_tokens": {
"model": MODEL,
"max_completion_tokens": 128,
"messages": [{"role": "user", "content": "hi"}],
},
"temperature_int_stays_int": {
"model": MODEL,
"temperature": 1,
"messages": [{"role": "user", "content": "hi"}],
},
"tools_auto": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"tool_choice": "auto",
"messages": [{"role": "user", "content": "Weather in Paris?"}],
},
"tools_strict": {
"model": MODEL,
"tools": [_STRICT_TOOL],
"messages": [{"role": "user", "content": "report this"}],
},
"tool_choice_required": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"tool_choice": "required",
"messages": [{"role": "user", "content": "Weather in Paris?"}],
},
"tool_choice_none": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"tool_choice": "none",
"messages": [{"role": "user", "content": "Weather in Paris?"}],
},
"tool_choice_specific": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
"messages": [{"role": "user", "content": "Weather in Paris?"}],
},
"parallel_tool_calls_false": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"parallel_tool_calls": False,
"messages": [{"role": "user", "content": "Weather in Paris and Rome?"}],
},
"tool_call_roundtrip": {
"model": MODEL,
"tools": [_WEATHER_TOOL],
"messages": [
{"role": "user", "content": "Weather in Paris?"},
_assistant_tool_call("call_1", "Paris"),
{"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 20C"},
],
},
"image_url_string_to_object": {
"model": MODEL,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "what is this"},
{"type": "image_url", "image_url": "https://e.test/a.png"},
],
}
],
},
"image_base64": {
"model": MODEL,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "and this"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
},
],
}
],
},
"response_format_json_object": {
"model": MODEL,
"response_format": {"type": "json_object"},
"messages": [{"role": "user", "content": "json please"}],
},
"response_format_json_schema_strict": {
"model": MODEL,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"capital": {"type": "string"}},
"required": ["capital"],
"additionalProperties": False,
},
"strict": True,
},
},
"messages": [{"role": "user", "content": "capital of France?"}],
},
"cache_control_stripped_everywhere": {
# v1 strips cache_control recursively from messages and tools; the IR
# carries it as typed metadata and the serializer drops it the same way.
"model": MODEL,
"tools": [
{
"type": "function",
"function": {
"name": "lookup",
"parameters": {"type": "object", "properties": {}},
"cache_control": {"type": "ephemeral"},
},
}
],
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "cached context",
"cache_control": {"type": "ephemeral"},
},
{"type": "text", "text": "question"},
],
}
],
},
}
# Typed fallbacks: each row must return a TranslationError whose summary
# carries the reason fragment, so the seam serves the request through v1.
# (v1 is NOT invoked for these rows: several would perform I/O, raise, or
# depend on get_optional_params interplay outside map_openai_params.)
EXPECTED_FALLBACKS = {
"o_series_model": (
{"model": "o3-mini", "messages": [{"role": "user", "content": "hi"}]},
"OpenAIOSeriesConfig",
),
"gpt5_model": (
{"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}]},
"OpenAIGPT5Config",
),
"http_pdf_file_id": (
{
"model": MODEL,
"messages": [
{
"role": "user",
"content": [
{
"type": "file",
"file": {"file_id": "https://e.test/doc.pdf"},
}
],
}
],
},
# the inbound schema has no file part at all, so v1's in-transform
# pdf download (gpt_transformation.py:236-257) can never be reached
"messages",
),
"stop_string_form": (
{"model": MODEL, "stop": "END", "messages": [{"role": "user", "content": "x"}]},
"string-form stop",
),
"both_max_tokens_keys": (
{
"model": MODEL,
"max_tokens": 5,
"max_completion_tokens": 6,
"messages": [{"role": "user", "content": "x"}],
},
"both max_tokens and max_completion_tokens",
),
"message_name_field": (
{
"model": MODEL,
"messages": [{"role": "user", "content": "x", "name": "alice"}],
},
"message name field",
),
"consecutive_user_messages": (
{
"model": MODEL,
"messages": [
{"role": "user", "content": "a"},
{"role": "user", "content": "b"},
],
},
"consecutive user messages",
),
"image_detail_key": (
{
"model": MODEL,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "see"},
{
"type": "image_url",
"image_url": {
"url": "https://e.test/a.png",
"detail": "low",
},
},
],
}
],
},
"image_url detail/format",
),
"tool_call_compact_arguments": (
{
"model": MODEL,
"messages": [
{"role": "user", "content": "w?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city":"Paris"}',
},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
],
},
"non-canonical JSON spacing",
),
"single_text_content_list": (
{
"model": MODEL,
"messages": [
{"role": "user", "content": [{"type": "text", "text": "only"}]}
],
},
"single-text content list",
),
"empty_tools_list": (
{"model": MODEL, "tools": [], "messages": [{"role": "user", "content": "x"}]},
"empty tools list",
),
"stream_options_unsupported": (
{
"model": MODEL,
"stream": True,
"stream_options": {"include_usage": True},
"messages": [{"role": "user", "content": "x"}],
},
"stream_options",
),
"user_param_model_list_gate": (
{"model": MODEL, "user": "u-1", "messages": [{"role": "user", "content": "x"}]},
"open_ai_chat_completion_models",
),
"top_k_not_openai": (
{"model": MODEL, "top_k": 40, "messages": [{"role": "user", "content": "x"}]},
"top_k",
),
"reasoning_effort_plain_gpt": (
{
"model": MODEL,
"reasoning_effort": "high",
"messages": [{"role": "user", "content": "x"}],
},
"reasoning_effort",
),
"response_format_on_gpt4": (
{
"model": "gpt-4",
"response_format": {"type": "json_object"},
"messages": [{"role": "user", "content": "x"}],
},
"outside v1's supported set",
),
"legacy_function_call": (
{
"model": MODEL,
"messages": [
{"role": "user", "content": "x"},
{"role": "assistant", "content": "", "function_call": {"name": "f"}},
],
},
"function_call",
),
}
def _v1_body(case: dict) -> dict:
request = copy.deepcopy(case)
config = OpenAIGPTConfig()
model = request["model"]
params = {
key: value for key, value in request.items() if key not in ("model", "messages")
}
optional = config.map_openai_params(
copy.deepcopy(params), {}, model, drop_params=False
)
return config.transform_request(
model, copy.deepcopy(request["messages"]), optional, {}, {}
)
def _v2_body(case: dict):
return translate_chat_request(
copy.deepcopy(case), "openai_compat", build_real_deps()
)
def _norm(body: dict) -> str:
return json.dumps(body, sort_keys=True, default=str)
@pytest.mark.parametrize("name", sorted(CORPUS))
def test_v2_request_matches_v1(name: str) -> None:
result = _v2_body(CORPUS[name])
assert result.is_ok(), result.error.summary
assert _norm(result.ok) == _norm(_v1_body(CORPUS[name]))
@pytest.mark.parametrize("name", sorted(EXPECTED_FALLBACKS))
def test_unsupported_shape_is_a_typed_fallback(name: str) -> None:
case, reason_fragment = EXPECTED_FALLBACKS[name]
result = _v2_body(case)
assert result.is_error(), f"{name} unexpectedly translated: {result.ok!r}"
assert reason_fragment in result.error.summary, result.error.summary
@@ -0,0 +1,285 @@
"""Differential parity for the openai response path.
The v1 reference is ``convert_to_model_response_object`` over the SDK-dump
response shape: the live normalizer on the SDK path (dossier gotcha #1;
``OpenAIGPTConfig.transform_response`` never runs there). v2 goes
``parse_response`` -> ``serialize_response(dialect="openai")`` ->
``to_model_response(usage_style="openai")``, and the two ``ModelResponse``
dumps must be identical: wire id/created/system_fingerprint survive, usage is
the verbatim ``Usage(**raw)`` passthrough (cached + reasoning details), the
finish_reason stop -> tool_calls rewrite fires, and reasoning content is
extracted from the key or ``<think>`` tags. Shapes the surface cannot trigger
must be typed errors, never silent drops.
"""
import copy
import json
import pytest
from litellm.types.utils import ModelResponse
from litellm.utils import convert_to_model_response_object
from litellm.translation.inbound.openai_chat import parse_request
from litellm.translation.inbound.openai_chat.response import serialize_response
from litellm.translation.providers.openai_compat.response import parse_response
from litellm.translation_seam import build_translation_deps, to_model_response
MODEL = "gpt-4o"
_REQUEST = {
"model": MODEL,
"messages": [{"role": "user", "content": "hi"}],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
},
}
],
}
_USAGE = {
"completion_tokens": 6,
"prompt_tokens": 12,
"total_tokens": 18,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0,
},
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0},
}
_RESPONSES = {
"text": {
"id": "chatcmpl-A1",
"object": "chat.completion",
"created": 1718000000,
"model": "gpt-4o-2024-08-06",
"system_fingerprint": "fp_abc123",
"service_tier": "default",
"choices": [
{
"index": 0,
"finish_reason": "stop",
"logprobs": None,
"message": {
"content": "Hello there.",
"role": "assistant",
"refusal": None,
"annotations": [],
"audio": None,
"function_call": None,
"tool_calls": None,
},
}
],
"usage": _USAGE,
},
"tool_calls_rewrites_stop": {
# finish_reason "stop" + tool_calls -> "tool_calls" on both sides
"id": "chatcmpl-T1",
"object": "chat.completion",
"created": 1718000001,
"model": "gpt-4o-2024-08-06",
"system_fingerprint": "fp_x",
"choices": [
{
"index": 0,
"finish_reason": "stop",
"logprobs": None,
"message": {
"content": None,
"role": "assistant",
"refusal": None,
"annotations": [],
"audio": None,
"function_call": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city":"Paris"}',
},
}
],
},
}
],
"usage": _USAGE,
},
"cached_and_reasoning_usage_details": {
"id": "chatcmpl-U1",
"object": "chat.completion",
"created": 1718000002,
"model": MODEL,
"choices": [
{
"index": 0,
"finish_reason": "length",
"logprobs": None,
"message": {
"content": "partial",
"role": "assistant",
"refusal": None,
"annotations": [],
},
}
],
"usage": {
"completion_tokens": 100,
"prompt_tokens": 1000,
"total_tokens": 1100,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 64,
"rejected_prediction_tokens": 0,
},
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 512},
},
},
"reasoning_content_key": {
# compat providers (deepseek-style) return reasoning_content beside
# content; v1 lifts it onto Message.reasoning_content
"id": "chatcmpl-R1",
"object": "chat.completion",
"created": 1718000003,
"model": MODEL,
"choices": [
{
"index": 0,
"finish_reason": "stop",
"logprobs": None,
"message": {
"content": "answer",
"role": "assistant",
"reasoning_content": "thought hard",
},
}
],
"usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12},
},
"think_tag_extraction": {
"id": "chatcmpl-K1",
"object": "chat.completion",
"created": 1718000004,
"model": MODEL,
"choices": [
{
"index": 0,
"finish_reason": "stop",
"logprobs": None,
"message": {"content": "<think>hmm</think>final", "role": "assistant"},
}
],
"usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12},
},
}
_UNSUPPORTED = {
"multiple_choices": (
{
"id": "chatcmpl-N",
"created": 1,
"model": MODEL,
"choices": [
{"index": 0, "finish_reason": "stop", "message": {"content": "a"}},
{"index": 1, "finish_reason": "stop", "message": {"content": "b"}},
],
"usage": _USAGE,
},
"multiple response choices",
),
"legacy_function_call_output": (
{
"id": "chatcmpl-F",
"created": 1,
"model": MODEL,
"choices": [
{
"index": 0,
"finish_reason": "function_call",
"message": {
"content": None,
"role": "assistant",
"function_call": {"name": "f", "arguments": "{}"},
},
}
],
"usage": _USAGE,
},
"function_call",
),
"multi_tool_use_parallel_repair": (
{
"id": "chatcmpl-M",
"created": 1,
"model": MODEL,
"choices": [
{
"index": 0,
"finish_reason": "tool_calls",
"message": {
"content": None,
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "multi_tool_use.parallel",
"arguments": '{"tool_uses": []}',
},
}
],
},
}
],
"usage": _USAGE,
},
"multi_tool_use.parallel",
),
}
def _v1_model_response(raw: dict) -> dict:
result = convert_to_model_response_object(
response_object=copy.deepcopy(raw), model_response_object=ModelResponse()
)
return result.model_dump()
def _v2_model_response(raw: dict) -> dict:
parsed = parse_request(copy.deepcopy(_REQUEST))
assert parsed.is_ok(), parsed.error.summary
response = parse_response(copy.deepcopy(raw), parsed.ok)
assert response.is_ok(), response.error.summary
body = serialize_response(response.ok, build_translation_deps(), "openai")
return to_model_response(body, usage_style="openai").model_dump()
def _norm(payload: dict) -> str:
return json.dumps(payload, sort_keys=True, default=str)
@pytest.mark.parametrize("name", sorted(_RESPONSES))
def test_v2_response_matches_v1(name: str, frozen_ambient) -> None:
raw = _RESPONSES[name]
assert _norm(_v2_model_response(raw)) == _norm(_v1_model_response(raw))
@pytest.mark.parametrize("name", sorted(_UNSUPPORTED))
def test_unreachable_response_shape_is_a_typed_error(name: str) -> None:
raw, reason_fragment = _UNSUPPORTED[name]
parsed = parse_request(copy.deepcopy(_REQUEST))
assert parsed.is_ok(), parsed.error.summary
result = parse_response(copy.deepcopy(raw), parsed.ok)
assert result.is_error(), f"{name} unexpectedly parsed"
assert reason_fragment in result.error.summary, result.error.summary
@@ -0,0 +1,259 @@
"""Differential parity for openai streaming, pinned at the SDK-chunk seam.
v1 side: recorded chunk dicts validated into the REAL SDK
``ChatCompletionChunk`` models and replayed through ``CustomStreamWrapper``
(custom_llm_provider="openai") the decode path production runs; SSE
framing is the OpenAI SDK's plumbing, exactly like AWS framing was
botocore's. v2 side: ``engine.stream.fold_events`` with the openai_compat
chunk parser and the ``openai`` chunk dialect. Chunk lists must be
byte-identical for content/tool/finish chunks.
The trailing usage chunk is the one pinned envelope difference: v1's wrapper
consumes the wire ``choices: []`` usage chunk into a SYNTHESIZED final chunk
(``stream_chunk_builder`` over its accumulated state, wrapper-cached model
string); the v2 fold passes the wire chunk through verbatim and the future
streaming seam owns that synthesis. The usage test pins both sides of that
contract: byte-identical prefix, equal usage numbers on the tail.
"""
import copy
import json
import time
import pytest
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.translation.engine.stream import fold_events, fold_lines
from litellm.translation.inbound.openai_chat.stream import initial_state
from litellm.translation.providers.openai_compat.stream import parse_event, parse_line
from litellm.translation_seam import to_model_response_stream
MODEL = "gpt-4o"
def _chunk(delta=None, finish=None, usage=None, choices=None):
payload = {
"id": "chatcmpl-S1",
"object": "chat.completion.chunk",
"created": 1718000000,
"model": "gpt-4o-2024-08-06",
"system_fingerprint": "fp_stream",
"service_tier": None,
"choices": [
{
"index": 0,
"delta": delta or {},
"logprobs": None,
"finish_reason": finish,
}
],
"usage": usage,
}
if choices is not None:
payload["choices"] = choices
return payload
def _delta(**overrides):
return {
"content": None,
"function_call": None,
"refusal": None,
"role": None,
"tool_calls": None,
**overrides,
}
STREAMS = {
"text": [
_chunk(_delta(role="assistant", content="")),
_chunk(_delta(content="Paris is")),
_chunk(_delta(content=" the capital.")),
_chunk(_delta(), finish="stop"),
],
"text_no_leading_role": [
# first content-bearing chunk still gains role: assistant
_chunk(_delta(content="Hi")),
_chunk(_delta(content=" there")),
_chunk(_delta(), finish="stop"),
],
"tools": [
_chunk(
_delta(
role="assistant",
tool_calls=[
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}
],
)
),
_chunk(
_delta(
tool_calls=[
{
"index": 0,
"id": None,
"type": None,
"function": {"name": None, "arguments": '{"ci'},
}
]
)
),
_chunk(
_delta(
tool_calls=[
{
"index": 0,
"id": None,
"type": None,
"function": {"name": None, "arguments": 'ty": "Paris"}'},
}
]
)
),
_chunk(_delta(), finish="tool_calls"),
],
"empty_keepalive_swallowed": [
_chunk(_delta(role="assistant", content="")),
_chunk(_delta()), # empty delta mid-stream: v1 drops it
_chunk(_delta(content="ok")),
_chunk(_delta(), finish="stop"),
],
}
_USAGE = {
"completion_tokens": 7,
"prompt_tokens": 11,
"total_tokens": 18,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0,
},
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0},
}
USAGE_STREAM = STREAMS["text"] + [_chunk(choices=[], usage=_USAGE)]
def _v1_chunks(events: list, stream_options=None) -> list:
logging = Logging(
model=MODEL,
messages=[{"role": "user", "content": "stream"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="diff-openai-stream",
function_id="diff-openai-stream",
)
sdk_chunks = (
ChatCompletionChunk.model_validate(event) for event in copy.deepcopy(events)
)
wrapper = CustomStreamWrapper(
completion_stream=sdk_chunks,
model=MODEL,
custom_llm_provider="openai",
logging_obj=logging,
stream_options=stream_options,
)
return [chunk.model_dump() for chunk in wrapper]
def _v2_chunks(events: list) -> list:
folded = fold_events(
copy.deepcopy(events), parse_event, initial_state(model=MODEL, dialect="openai")
)
assert folded.is_ok(), folded.error.summary
return [
to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump()
for chunk in folded.ok
]
def _norm(chunks: list) -> str:
return json.dumps(chunks, sort_keys=True, default=str)
@pytest.mark.parametrize("name", sorted(STREAMS))
def test_v2_stream_matches_v1(name: str, frozen_ambient) -> None:
events = STREAMS[name]
assert _norm(_v2_chunks(events)) == _norm(_v1_chunks(events))
def test_v2_stream_decodes_sse_lines_identically(frozen_ambient) -> None:
"""fold_lines over the raw SSE framing (data: ... / [DONE]) produces the
same chunks as fold_events over the parsed payloads."""
events = STREAMS["text"]
lines = [f"data: {json.dumps(event)}" for event in events] + ["", "data: [DONE]"]
folded = fold_lines(lines, parse_line, initial_state(model=MODEL, dialect="openai"))
assert folded.is_ok(), folded.error.summary
via_lines = [
to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump()
for chunk in folded.ok
]
assert _norm(via_lines) == _norm(_v2_chunks(events))
def test_usage_chunk_passthrough_pins_the_seam_contract(frozen_ambient) -> None:
v1 = _v1_chunks(USAGE_STREAM, stream_options={"include_usage": True})
v2 = _v2_chunks(USAGE_STREAM)
# content + finish chunks are byte-identical
assert _norm(v2[:-1]) == _norm(v1[: len(v2) - 1])
# v1's tail is the wrapper-synthesized usage chunk (envelope); v2's tail
# is the wire usage chunk verbatim. The usage numbers must agree.
assert len(v1) == len(v2)
v1_tail, v2_tail = v1[-1], v2[-1]
assert v2_tail["choices"] == []
assert v1_tail["usage"] is not None and v2_tail["usage"] is not None
for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
assert v1_tail["usage"][key] == v2_tail["usage"][key] == _USAGE[key]
assert (
v2_tail["usage"]["prompt_tokens_details"]["cached_tokens"]
== _USAGE["prompt_tokens_details"]["cached_tokens"]
)
assert (
v2_tail["usage"]["completion_tokens_details"]["reasoning_tokens"]
== _USAGE["completion_tokens_details"]["reasoning_tokens"]
)
_UNSUPPORTED_CHUNKS = {
"function_call_delta": (
_chunk(_delta(function_call={"name": "f", "arguments": ""})),
"function_call",
),
"unknown_finish_reason": (
_chunk(_delta(), finish="function_call"),
"finish_reason",
),
"multiple_choices": (
_chunk(
choices=[
{"index": 0, "delta": _delta(content="a"), "finish_reason": None},
{"index": 1, "delta": _delta(content="b"), "finish_reason": None},
]
),
"multiple stream choices",
),
"unknown_delta_key": (
_chunk({**_delta(content="x"), "reasoning_content": "hmm"}),
"stream delta keys",
),
}
@pytest.mark.parametrize("name", sorted(_UNSUPPORTED_CHUNKS))
def test_unreachable_chunk_shape_is_a_typed_error(name: str) -> None:
event, reason_fragment = _UNSUPPORTED_CHUNKS[name]
result = parse_event(event)
assert result.is_error(), f"{name} unexpectedly parsed"
assert reason_fragment in result.error.summary, result.error.summary