diff --git a/litellm/translation/CLAUDE.md b/litellm/translation/CLAUDE.md index 7bc1c707ab..a88b3e4f9f 100644 --- a/litellm/translation/CLAUDE.md +++ b/litellm/translation/CLAUDE.md @@ -55,13 +55,27 @@ translation/ │ │ ├── response.py # converse JSON -> IR (properties-unwrap json rewrite) │ │ └── stream.py # PARSED converse events -> IR events (pinned at the │ │ # parsed-event seam; AWS framing is botocore's) -│ └── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY: -│ ├── serialize.py # pop model/stream, inject anthropic_version, spoof -│ │ # model for response_format (v1's json-tool forcing) -│ ├── response.py # re-export of anthropic parse_response (invoke -│ │ # response body IS anthropic wire format) -│ └── stream.py # re-export of anthropic parse_event (invoke stream -│ # = anthropic events over AWS framing) +│ ├── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY: +│ │ ├── serialize.py # pop model/stream, inject anthropic_version, spoof +│ │ │ # model for response_format (v1's json-tool forcing) +│ │ ├── response.py # re-export of anthropic parse_response (invoke +│ │ │ # response body IS anthropic wire format) +│ │ └── stream.py # re-export of anthropic parse_event (invoke stream +│ │ # = anthropic events over AWS framing) +│ └── openai_compat/ # the same-family hub serializer (GPT first consumer) +│ ├── guard.py # raw-shape fidelity guard run BEFORE parse: shapes +│ │ # the IR cannot round-trip losslessly (string stop, +│ │ # message name, image detail, max-tokens key split, +│ │ # tool-arg spacing) fall back to v1 as typed errors +│ ├── serialize.py # v1's five-touch passthrough body assembly +│ ├── messages.py # IR -> openai wire messages (inverse of inbound) +│ ├── params.py # o-series/gpt-5 family gates (fail closed until +│ │ # their param families are ported), user gate +│ ├── response.py # mirrors convert_to_model_response_object (the LIVE +│ │ # normalizer; transform_response is dead on the SDK +│ │ # path); rides the outbound body on ChatResponse.wire +│ └── stream.py # SSE chunk -> wire_chunk events normalized to the +│ # SDK-dump shape; the openai chunk dialect folds them └── engine/ ├── pipeline.py # prepare (pure, drives the fallback decision) -> send; │ # per-provider serializer/parser/dialect tables; the @@ -178,22 +192,36 @@ A behavior change ships as its own snapshot-diffed PR, never inside a port. ## Current scope -OpenAI-chat-in to three providers out — `anthropic`, `bedrock_converse`, -`bedrock_invoke` — request, response, and stream translation, -differential-green (anthropic: 46-shape corpus + responses + stream replays; -bedrock: the characterization corpus per route + quirk corpus), fail-closed -everywhere else, with non-streaming flag-gated seams live in `completion()`. +OpenAI-chat-in to four providers out — `anthropic`, `bedrock_converse`, +`bedrock_invoke`, `openai_compat` — request, response, and stream +translation, differential-green (anthropic: 46-shape corpus + responses + +stream replays; bedrock: the characterization corpus per route + quirk +corpus; openai: 17-shape request corpus + 17 typed-fallback rows + response +and SDK-chunk stream replays), fail-closed everywhere else, with +non-streaming flag-gated seams live in `completion()` for the anthropic and +bedrock routes (the openai seam fork is integrator scope and NOT wired). Deliberate bedrock fallback surfaces (each names the v1 path): non-Claude bedrock models, native structured outputs (outputConfig), adaptive-effort output_config/beta, response_format+stream (fake_stream), response_format with thinking on invoke (the model spoof crossing), tool history without tools (modify_params dummy tool), empty user text on converse (string-vs-list ambiguity), provisioned `model_id`, `guardrailConfig`, and -``-tagged text in converse responses. Not yet here, each its own -follow-up: streaming seams live; the other inbound schemas -(`anthropic_messages`, `google_genai`, `responses`, `completions`); the other -providers (vertex, azure, `openai_compat`); the same-family fast path -(waits on the opaque-body relay). To add a provider: write +``-tagged text in converse responses. Deliberate openai fallback +surfaces: o-series and gpt-5 model families (their param-rewrite configs are +unported), every raw shape the IR cannot round-trip byte-identically (the +guard's list: string stop, both max-tokens keys, message `name`, image +`detail`/`format`, consecutive same-role turns, single-text content lists, +empty tools/stop lists, non-canonical tool-argument JSON spacing, null/ +non-function tool_calls), the `user` param (model-list gated in v1), +`response_format` on gpt-4/gpt-3.5-turbo-16k, `stream_options`, file blocks +(v1 downloads http pdf file_ids in-transform), and `http://` image URLs. On +streams, the trailing `choices: []` usage chunk passes through verbatim and +the wrapper's synthesized final usage chunk stays a seam/envelope concern. +Not yet here, each its own follow-up: streaming seams live; the other +inbound schemas (`anthropic_messages`, `google_genai`, `responses`, +`completions`); the other providers (vertex, azure); the same-family fast +path (waits on the opaque-body relay). To add a provider: write `providers//`, register it in `engine/pipeline._SERIALIZERS` / -`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS`, add a differential corpus, keep -the flag off until differential-green. +`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS` (plus `_RAW_GUARDS` when the +inbound schema is the provider's own family), add a differential corpus, +keep the flag off until differential-green. diff --git a/litellm/translation/engine/pipeline.py b/litellm/translation/engine/pipeline.py index eea7510b96..aee590e39b 100644 --- a/litellm/translation/engine/pipeline.py +++ b/litellm/translation/engine/pipeline.py @@ -39,6 +39,13 @@ from ..providers.bedrock_invoke import parse_response as bedrock_invoke_parse_re from ..providers.bedrock_invoke import ( serialize_request as bedrock_invoke_serialize_request, ) +from ..providers.openai_compat import parse_response as openai_compat_parse_response +from ..providers.openai_compat import ( + serialize_request as openai_compat_serialize_request, +) +from ..providers.openai_compat import ( + unsupported_request_shapes as openai_compat_unsupported_request_shapes, +) from .http import Endpoint, ExecuteError, HttpPort, ProviderHttpError _Serializer = Callable[[ChatRequest, TranslationDeps], Result[Body, TranslationError]] @@ -51,6 +58,7 @@ _SERIALIZERS: Mapping[Provider, _Serializer] = MappingProxyType( "anthropic": serialize_request, "bedrock_converse": bedrock_converse_serialize_request, "bedrock_invoke": bedrock_invoke_serialize_request, + "openai_compat": openai_compat_serialize_request, } ) @@ -59,6 +67,7 @@ _RESPONSE_PARSERS: Mapping[Provider, _ResponseParser] = MappingProxyType( "anthropic": parse_response, "bedrock_converse": bedrock_converse_parse_response, "bedrock_invoke": bedrock_invoke_parse_response, + "openai_compat": openai_compat_parse_response, } ) @@ -67,9 +76,28 @@ _RESPONSE_DIALECTS: Mapping[Provider, ResponseDialect] = MappingProxyType( "anthropic": "anthropic", "bedrock_converse": "bedrock_converse", "bedrock_invoke": "anthropic", # invoke delegates to the anthropic transform + "openai_compat": "openai", # same-family: the wire-derived body } ) +_RawGuard = Callable[[Mapping[str, object]], TranslationError | None] + +_RAW_GUARDS: Mapping[Provider, _RawGuard] = MappingProxyType( + # Same-family providers run a raw-shape fidelity guard BEFORE parse: the + # inbound parse normalizes wire forms v1 forwards verbatim, so shapes it + # cannot round-trip losslessly fall back to v1 as typed errors. + { + "openai_compat": openai_compat_unsupported_request_shapes, + } +) + + +def _raw_guard_error( + raw: Mapping[str, object], provider: Provider +) -> TranslationError | None: + guard = _RAW_GUARDS.get(provider) + return guard(raw) if guard is not None else None + def response_dialect(provider: Provider) -> ResponseDialect: return _RESPONSE_DIALECTS.get(provider, "anthropic") @@ -85,6 +113,9 @@ def translate_chat_request( f"provider {provider!r} has no v2 chat serializer yet" ) ) + guard_error = _raw_guard_error(raw, provider) + if guard_error is not None: + return Error(guard_error) return parse_request(raw).bind(lambda request: serializer(request, deps)) @@ -127,6 +158,9 @@ def prepare_chat_request( f"provider {provider!r} is not fully ported to v2 yet" ) ) + guard_error = _raw_guard_error(raw, provider) + if guard_error is not None: + return Error(guard_error) match parse_request(raw): case Result(tag="ok", ok=request): pass diff --git a/litellm/translation/providers/openai_compat/__init__.py b/litellm/translation/providers/openai_compat/__init__.py new file mode 100644 index 0000000000..489340941f --- /dev/null +++ b/litellm/translation/providers/openai_compat/__init__.py @@ -0,0 +1,12 @@ +from .guard import unsupported_request_shapes +from .response import parse_response +from .serialize import serialize_request +from .stream import parse_event, parse_line + +__all__ = ( + "parse_event", + "parse_line", + "parse_response", + "serialize_request", + "unsupported_request_shapes", +) diff --git a/litellm/translation/providers/openai_compat/serialize.py b/litellm/translation/providers/openai_compat/serialize.py index a276722df4..ffec4aae40 100644 --- a/litellm/translation/providers/openai_compat/serialize.py +++ b/litellm/translation/providers/openai_compat/serialize.py @@ -16,9 +16,10 @@ from __future__ import annotations from expression import Error, Ok, Option, Result from expression.collections import Block -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from typing_extensions import assert_never +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH + from ...deps import TranslationDeps from ...errors import TranslationError from ...ir import Body, ChatRequest, PlainJson, ResponseFormat, ToolChoice, ToolDef diff --git a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md index ec821c1b6e..3662ddfd71 100644 --- a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md +++ b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md @@ -1,4 +1,4 @@ -# Translation v2 differential report (anthropic + bedrock) +# Translation v2 differential report (anthropic + bedrock + openai) v1 and v2 run over the same corpus; every row must be IDENTICAL (or an explained FALLBACK that v1 serves) for a provider's flag to turn on. @@ -6,7 +6,7 @@ Bedrock rows additionally pin the characterization-corpus snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: `python -m tests.test_litellm.translation.generate_differential_report` -- commit: a10779c756 +- commit: c1fee272a0 ## anthropic: request bodies (v1 map_openai_params + transform_request vs v2) @@ -76,6 +76,59 @@ each row proves snapshot == v1-at-HEAD == v2. Regenerate with: - IDENTICAL: thinking - IDENTICAL: tools +## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2) + +- IDENTICAL: cache_control_stripped_everywhere +- IDENTICAL: image_base64 +- IDENTICAL: image_url_string_to_object +- IDENTICAL: max_completion_tokens +- IDENTICAL: multiturn_stop_list_stream +- IDENTICAL: parallel_tool_calls_false +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema_strict +- IDENTICAL: system_and_sampling +- IDENTICAL: temperature_int_stays_int +- IDENTICAL: text +- IDENTICAL: tool_call_roundtrip +- IDENTICAL: tool_choice_none +- IDENTICAL: tool_choice_required +- IDENTICAL: tool_choice_specific +- IDENTICAL: tools_auto +- IDENTICAL: tools_strict +- FALLBACK (v1 serves it): both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): consecutive_user_messages (consecutive user messages) +- FALLBACK (v1 serves it): empty_tools_list (empty tools list) +- FALLBACK (v1 serves it): gpt5_model (OpenAIGPT5Config) +- FALLBACK (v1 serves it): http_pdf_file_id (messages) +- FALLBACK (v1 serves it): image_detail_key (image_url detail/format) +- FALLBACK (v1 serves it): legacy_function_call (function_call) +- FALLBACK (v1 serves it): message_name_field (message name field) +- FALLBACK (v1 serves it): o_series_model (OpenAIOSeriesConfig) +- FALLBACK (v1 serves it): reasoning_effort_plain_gpt (reasoning_effort) +- FALLBACK (v1 serves it): response_format_on_gpt4 (outside v1's supported set) +- FALLBACK (v1 serves it): single_text_content_list (single-text content list) +- FALLBACK (v1 serves it): stop_string_form (string-form stop) +- FALLBACK (v1 serves it): stream_options_unsupported (stream_options) +- FALLBACK (v1 serves it): tool_call_compact_arguments (non-canonical JSON spacing) +- FALLBACK (v1 serves it): top_k_not_openai (top_k) +- FALLBACK (v1 serves it): user_param_model_list_gate (open_ai_chat_completion_models) + +## openai_compat: responses (v1 convert_to_model_response_object vs v2) + +- IDENTICAL: cached_and_reasoning_usage_details +- IDENTICAL: reasoning_content_key +- IDENTICAL: text +- IDENTICAL: think_tag_extraction +- IDENTICAL: tool_calls_rewrites_stop + +## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold) + +- IDENTICAL: empty_keepalive_swallowed +- IDENTICAL: text +- IDENTICAL: text_no_leading_role +- IDENTICAL: tools +- SEAM CONTRACT: usage tail (v2 passes the wire choices=[] usage chunk through; v1's wrapper synthesizes its final usage chunk from it, which is the streaming seam's envelope to reproduce) + ## bedrock_converse: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON) - IDENTICAL: cache_control_messages diff --git a/tests/test_litellm/translation/generate_differential_report.py b/tests/test_litellm/translation/generate_differential_report.py index 13bcbf86ad..82834b4e32 100644 --- a/tests/test_litellm/translation/generate_differential_report.py +++ b/tests/test_litellm/translation/generate_differential_report.py @@ -63,6 +63,77 @@ def _anthropic_rows(lines: list) -> int: return failures +def _openai_rows(lines: list) -> int: + from litellm.translation import translate_chat_request + + from . import test_differential_openai_request as req + from . import test_differential_openai_response as resp + from . import test_differential_openai_stream as stream + from .conftest import build_real_deps + + failures = 0 + lines += [ + "", + "## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2)", + "", + ] + for name in sorted(req.CORPUS): + result = req._v2_body(req.CORPUS[name]) + same = result.is_ok() and req._norm(result.ok) == req._norm( + req._v1_body(req.CORPUS[name]) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + for name in sorted(req.EXPECTED_FALLBACKS): + case, reason = req.EXPECTED_FALLBACKS[name] + result = translate_chat_request(dict(case), "openai_compat", build_real_deps()) + ok = result.is_error() and reason in result.error.summary + failures += 0 if ok else 1 + label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT" + lines.append(f"- {label}: {name} ({reason})") + lines += [ + "", + "## openai_compat: responses (v1 convert_to_model_response_object vs v2)", + "", + ] + for name in sorted(resp._RESPONSES): + same = resp._norm(resp._v2_model_response(resp._RESPONSES[name])) == resp._norm( + resp._v1_model_response(resp._RESPONSES[name]) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + lines += [ + "", + "## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold)", + "", + ] + for name in sorted(stream.STREAMS): + same = stream._norm(stream._v2_chunks(stream.STREAMS[name])) == stream._norm( + stream._v1_chunks(stream.STREAMS[name]) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + v1 = stream._v1_chunks(stream.USAGE_STREAM, stream_options={"include_usage": True}) + v2 = stream._v2_chunks(stream.USAGE_STREAM) + tail_ok = ( + len(v1) == len(v2) + and stream._norm(v2[:-1]) == stream._norm(v1[: len(v2) - 1]) + and v2[-1]["choices"] == [] + and all( + v1[-1]["usage"][k] == v2[-1]["usage"][k] + for k in ("prompt_tokens", "completion_tokens", "total_tokens") + ) + ) + failures += 0 if tail_ok else 1 + lines.append( + ("- SEAM CONTRACT: " if tail_ok else "- DIVERGENT: ") + + "usage tail (v2 passes the wire choices=[] usage chunk through; v1's" + " wrapper synthesizes its final usage chunk from it, which is the" + " streaming seam's envelope to reproduce)" + ) + return failures + + def _bedrock_request_rows(lines: list) -> int: from litellm.translation import translate_chat_request @@ -195,7 +266,7 @@ def main() -> None: _freeze_ambient() lines = [ - "# Translation v2 differential report (anthropic + bedrock)", + "# Translation v2 differential report (anthropic + bedrock + openai)", "", "v1 and v2 run over the same corpus; every row must be IDENTICAL (or an", "explained FALLBACK that v1 serves) for a provider's flag to turn on.", @@ -207,6 +278,7 @@ def main() -> None: "", ] failures = _anthropic_rows(lines) + failures += _openai_rows(lines) failures += _bedrock_request_rows(lines) failures += _bedrock_response_rows(lines) failures += _bedrock_stream_rows(lines) diff --git a/tests/test_litellm/translation/test_differential_openai_request.py b/tests/test_litellm/translation/test_differential_openai_request.py new file mode 100644 index 0000000000..f6748b4289 --- /dev/null +++ b/tests/test_litellm/translation/test_differential_openai_request.py @@ -0,0 +1,432 @@ +"""Differential parity: v2 IR translation vs the v1 OpenAIGPTConfig chain. + +For each OpenAI chat request in the corpus, the v1 body is produced exactly as +``litellm.completion`` would for provider "openai" (``map_openai_params`` then +``transform_request``) and compared, as normalized JSON, to the v2 body from +``translate_chat_request``. The corpus is the covered surface: the +openai_compat v2 flag only turns on for the shapes pinned here. v1 is a +near-passthrough (five touches), so byte parity is only possible for shapes +the inbound parse round-trips losslessly; everything else must be a TYPED +fallback (the raw guard / parse / serializer reasons asserted below), never a +silent divergence. +""" + +import copy +import json + +import pytest + +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.translation import translate_chat_request + +from .conftest import build_real_deps + +MODEL = "gpt-4o" + +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} + +_STRICT_TOOL = { + "type": "function", + "function": { + "name": "report", + "parameters": { + "type": "object", + "properties": {"body": {"type": "string"}}, + "required": ["body"], + "additionalProperties": False, + }, + "strict": True, + }, +} + + +def _assistant_tool_call(call_id, city, name="get_weather"): + return { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + # json.dumps default spacing: the only argument form the IR + # re-dumps byte-identically (the guard rejects the rest) + "function": {"name": name, "arguments": json.dumps({"city": city})}, + } + ], + } + + +CORPUS = { + "text": { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello, world"}], + }, + "system_and_sampling": { + "model": MODEL, + "max_tokens": 50, + "temperature": 0.5, + "top_p": 0.9, + "messages": [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "Hi"}, + ], + }, + "multiturn_stop_list_stream": { + "model": MODEL, + "max_tokens": 64, + "stop": ["END", "STOP"], + "stream": True, + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + {"role": "user", "content": "How are you?"}, + ], + }, + "max_completion_tokens": { + "model": MODEL, + "max_completion_tokens": 128, + "messages": [{"role": "user", "content": "hi"}], + }, + "temperature_int_stays_int": { + "model": MODEL, + "temperature": 1, + "messages": [{"role": "user", "content": "hi"}], + }, + "tools_auto": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "tool_choice": "auto", + "messages": [{"role": "user", "content": "Weather in Paris?"}], + }, + "tools_strict": { + "model": MODEL, + "tools": [_STRICT_TOOL], + "messages": [{"role": "user", "content": "report this"}], + }, + "tool_choice_required": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "tool_choice": "required", + "messages": [{"role": "user", "content": "Weather in Paris?"}], + }, + "tool_choice_none": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "tool_choice": "none", + "messages": [{"role": "user", "content": "Weather in Paris?"}], + }, + "tool_choice_specific": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + }, + "parallel_tool_calls_false": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "parallel_tool_calls": False, + "messages": [{"role": "user", "content": "Weather in Paris and Rome?"}], + }, + "tool_call_roundtrip": { + "model": MODEL, + "tools": [_WEATHER_TOOL], + "messages": [ + {"role": "user", "content": "Weather in Paris?"}, + _assistant_tool_call("call_1", "Paris"), + {"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 20C"}, + ], + }, + "image_url_string_to_object": { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": "https://e.test/a.png"}, + ], + } + ], + }, + "image_base64": { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "and this"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ], + } + ], + }, + "response_format_json_object": { + "model": MODEL, + "response_format": {"type": "json_object"}, + "messages": [{"role": "user", "content": "json please"}], + }, + "response_format_json_schema_strict": { + "model": MODEL, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"capital": {"type": "string"}}, + "required": ["capital"], + "additionalProperties": False, + }, + "strict": True, + }, + }, + "messages": [{"role": "user", "content": "capital of France?"}], + }, + "cache_control_stripped_everywhere": { + # v1 strips cache_control recursively from messages and tools; the IR + # carries it as typed metadata and the serializer drops it the same way. + "model": MODEL, + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + }, + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cached context", + "cache_control": {"type": "ephemeral"}, + }, + {"type": "text", "text": "question"}, + ], + } + ], + }, +} + +# Typed fallbacks: each row must return a TranslationError whose summary +# carries the reason fragment, so the seam serves the request through v1. +# (v1 is NOT invoked for these rows: several would perform I/O, raise, or +# depend on get_optional_params interplay outside map_openai_params.) +EXPECTED_FALLBACKS = { + "o_series_model": ( + {"model": "o3-mini", "messages": [{"role": "user", "content": "hi"}]}, + "OpenAIOSeriesConfig", + ), + "gpt5_model": ( + {"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}]}, + "OpenAIGPT5Config", + ), + "http_pdf_file_id": ( + { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_id": "https://e.test/doc.pdf"}, + } + ], + } + ], + }, + # the inbound schema has no file part at all, so v1's in-transform + # pdf download (gpt_transformation.py:236-257) can never be reached + "messages", + ), + "stop_string_form": ( + {"model": MODEL, "stop": "END", "messages": [{"role": "user", "content": "x"}]}, + "string-form stop", + ), + "both_max_tokens_keys": ( + { + "model": MODEL, + "max_tokens": 5, + "max_completion_tokens": 6, + "messages": [{"role": "user", "content": "x"}], + }, + "both max_tokens and max_completion_tokens", + ), + "message_name_field": ( + { + "model": MODEL, + "messages": [{"role": "user", "content": "x", "name": "alice"}], + }, + "message name field", + ), + "consecutive_user_messages": ( + { + "model": MODEL, + "messages": [ + {"role": "user", "content": "a"}, + {"role": "user", "content": "b"}, + ], + }, + "consecutive user messages", + ), + "image_detail_key": ( + { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "see"}, + { + "type": "image_url", + "image_url": { + "url": "https://e.test/a.png", + "detail": "low", + }, + }, + ], + } + ], + }, + "image_url detail/format", + ), + "tool_call_compact_arguments": ( + { + "model": MODEL, + "messages": [ + {"role": "user", "content": "w?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city":"Paris"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, + ], + }, + "non-canonical JSON spacing", + ), + "single_text_content_list": ( + { + "model": MODEL, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "only"}]} + ], + }, + "single-text content list", + ), + "empty_tools_list": ( + {"model": MODEL, "tools": [], "messages": [{"role": "user", "content": "x"}]}, + "empty tools list", + ), + "stream_options_unsupported": ( + { + "model": MODEL, + "stream": True, + "stream_options": {"include_usage": True}, + "messages": [{"role": "user", "content": "x"}], + }, + "stream_options", + ), + "user_param_model_list_gate": ( + {"model": MODEL, "user": "u-1", "messages": [{"role": "user", "content": "x"}]}, + "open_ai_chat_completion_models", + ), + "top_k_not_openai": ( + {"model": MODEL, "top_k": 40, "messages": [{"role": "user", "content": "x"}]}, + "top_k", + ), + "reasoning_effort_plain_gpt": ( + { + "model": MODEL, + "reasoning_effort": "high", + "messages": [{"role": "user", "content": "x"}], + }, + "reasoning_effort", + ), + "response_format_on_gpt4": ( + { + "model": "gpt-4", + "response_format": {"type": "json_object"}, + "messages": [{"role": "user", "content": "x"}], + }, + "outside v1's supported set", + ), + "legacy_function_call": ( + { + "model": MODEL, + "messages": [ + {"role": "user", "content": "x"}, + {"role": "assistant", "content": "", "function_call": {"name": "f"}}, + ], + }, + "function_call", + ), +} + + +def _v1_body(case: dict) -> dict: + request = copy.deepcopy(case) + config = OpenAIGPTConfig() + model = request["model"] + params = { + key: value for key, value in request.items() if key not in ("model", "messages") + } + optional = config.map_openai_params( + copy.deepcopy(params), {}, model, drop_params=False + ) + return config.transform_request( + model, copy.deepcopy(request["messages"]), optional, {}, {} + ) + + +def _v2_body(case: dict): + return translate_chat_request( + copy.deepcopy(case), "openai_compat", build_real_deps() + ) + + +def _norm(body: dict) -> str: + return json.dumps(body, sort_keys=True, default=str) + + +@pytest.mark.parametrize("name", sorted(CORPUS)) +def test_v2_request_matches_v1(name: str) -> None: + result = _v2_body(CORPUS[name]) + assert result.is_ok(), result.error.summary + assert _norm(result.ok) == _norm(_v1_body(CORPUS[name])) + + +@pytest.mark.parametrize("name", sorted(EXPECTED_FALLBACKS)) +def test_unsupported_shape_is_a_typed_fallback(name: str) -> None: + case, reason_fragment = EXPECTED_FALLBACKS[name] + result = _v2_body(case) + assert result.is_error(), f"{name} unexpectedly translated: {result.ok!r}" + assert reason_fragment in result.error.summary, result.error.summary diff --git a/tests/test_litellm/translation/test_differential_openai_response.py b/tests/test_litellm/translation/test_differential_openai_response.py new file mode 100644 index 0000000000..94602bb3fb --- /dev/null +++ b/tests/test_litellm/translation/test_differential_openai_response.py @@ -0,0 +1,285 @@ +"""Differential parity for the openai response path. + +The v1 reference is ``convert_to_model_response_object`` over the SDK-dump +response shape: the live normalizer on the SDK path (dossier gotcha #1; +``OpenAIGPTConfig.transform_response`` never runs there). v2 goes +``parse_response`` -> ``serialize_response(dialect="openai")`` -> +``to_model_response(usage_style="openai")``, and the two ``ModelResponse`` +dumps must be identical: wire id/created/system_fingerprint survive, usage is +the verbatim ``Usage(**raw)`` passthrough (cached + reasoning details), the +finish_reason stop -> tool_calls rewrite fires, and reasoning content is +extracted from the key or ```` tags. Shapes the surface cannot trigger +must be typed errors, never silent drops. +""" + +import copy +import json + +import pytest + +from litellm.types.utils import ModelResponse +from litellm.utils import convert_to_model_response_object + +from litellm.translation.inbound.openai_chat import parse_request +from litellm.translation.inbound.openai_chat.response import serialize_response +from litellm.translation.providers.openai_compat.response import parse_response +from litellm.translation_seam import build_translation_deps, to_model_response + +MODEL = "gpt-4o" + +_REQUEST = { + "model": MODEL, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], +} + +_USAGE = { + "completion_tokens": 6, + "prompt_tokens": 12, + "total_tokens": 18, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, +} + +_RESPONSES = { + "text": { + "id": "chatcmpl-A1", + "object": "chat.completion", + "created": 1718000000, + "model": "gpt-4o-2024-08-06", + "system_fingerprint": "fp_abc123", + "service_tier": "default", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "logprobs": None, + "message": { + "content": "Hello there.", + "role": "assistant", + "refusal": None, + "annotations": [], + "audio": None, + "function_call": None, + "tool_calls": None, + }, + } + ], + "usage": _USAGE, + }, + "tool_calls_rewrites_stop": { + # finish_reason "stop" + tool_calls -> "tool_calls" on both sides + "id": "chatcmpl-T1", + "object": "chat.completion", + "created": 1718000001, + "model": "gpt-4o-2024-08-06", + "system_fingerprint": "fp_x", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "logprobs": None, + "message": { + "content": None, + "role": "assistant", + "refusal": None, + "annotations": [], + "audio": None, + "function_call": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city":"Paris"}', + }, + } + ], + }, + } + ], + "usage": _USAGE, + }, + "cached_and_reasoning_usage_details": { + "id": "chatcmpl-U1", + "object": "chat.completion", + "created": 1718000002, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "length", + "logprobs": None, + "message": { + "content": "partial", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + } + ], + "usage": { + "completion_tokens": 100, + "prompt_tokens": 1000, + "total_tokens": 1100, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 64, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 512}, + }, + }, + "reasoning_content_key": { + # compat providers (deepseek-style) return reasoning_content beside + # content; v1 lifts it onto Message.reasoning_content + "id": "chatcmpl-R1", + "object": "chat.completion", + "created": 1718000003, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "logprobs": None, + "message": { + "content": "answer", + "role": "assistant", + "reasoning_content": "thought hard", + }, + } + ], + "usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12}, + }, + "think_tag_extraction": { + "id": "chatcmpl-K1", + "object": "chat.completion", + "created": 1718000004, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "logprobs": None, + "message": {"content": "hmmfinal", "role": "assistant"}, + } + ], + "usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12}, + }, +} + +_UNSUPPORTED = { + "multiple_choices": ( + { + "id": "chatcmpl-N", + "created": 1, + "model": MODEL, + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"content": "a"}}, + {"index": 1, "finish_reason": "stop", "message": {"content": "b"}}, + ], + "usage": _USAGE, + }, + "multiple response choices", + ), + "legacy_function_call_output": ( + { + "id": "chatcmpl-F", + "created": 1, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "function_call", + "message": { + "content": None, + "role": "assistant", + "function_call": {"name": "f", "arguments": "{}"}, + }, + } + ], + "usage": _USAGE, + }, + "function_call", + ), + "multi_tool_use_parallel_repair": ( + { + "id": "chatcmpl-M", + "created": 1, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "multi_tool_use.parallel", + "arguments": '{"tool_uses": []}', + }, + } + ], + }, + } + ], + "usage": _USAGE, + }, + "multi_tool_use.parallel", + ), +} + + +def _v1_model_response(raw: dict) -> dict: + result = convert_to_model_response_object( + response_object=copy.deepcopy(raw), model_response_object=ModelResponse() + ) + return result.model_dump() + + +def _v2_model_response(raw: dict) -> dict: + parsed = parse_request(copy.deepcopy(_REQUEST)) + assert parsed.is_ok(), parsed.error.summary + response = parse_response(copy.deepcopy(raw), parsed.ok) + assert response.is_ok(), response.error.summary + body = serialize_response(response.ok, build_translation_deps(), "openai") + return to_model_response(body, usage_style="openai").model_dump() + + +def _norm(payload: dict) -> str: + return json.dumps(payload, sort_keys=True, default=str) + + +@pytest.mark.parametrize("name", sorted(_RESPONSES)) +def test_v2_response_matches_v1(name: str, frozen_ambient) -> None: + raw = _RESPONSES[name] + assert _norm(_v2_model_response(raw)) == _norm(_v1_model_response(raw)) + + +@pytest.mark.parametrize("name", sorted(_UNSUPPORTED)) +def test_unreachable_response_shape_is_a_typed_error(name: str) -> None: + raw, reason_fragment = _UNSUPPORTED[name] + parsed = parse_request(copy.deepcopy(_REQUEST)) + assert parsed.is_ok(), parsed.error.summary + result = parse_response(copy.deepcopy(raw), parsed.ok) + assert result.is_error(), f"{name} unexpectedly parsed" + assert reason_fragment in result.error.summary, result.error.summary diff --git a/tests/test_litellm/translation/test_differential_openai_stream.py b/tests/test_litellm/translation/test_differential_openai_stream.py new file mode 100644 index 0000000000..26cdc2c28e --- /dev/null +++ b/tests/test_litellm/translation/test_differential_openai_stream.py @@ -0,0 +1,259 @@ +"""Differential parity for openai streaming, pinned at the SDK-chunk seam. + +v1 side: recorded chunk dicts validated into the REAL SDK +``ChatCompletionChunk`` models and replayed through ``CustomStreamWrapper`` +(custom_llm_provider="openai") — the decode path production runs; SSE +framing is the OpenAI SDK's plumbing, exactly like AWS framing was +botocore's. v2 side: ``engine.stream.fold_events`` with the openai_compat +chunk parser and the ``openai`` chunk dialect. Chunk lists must be +byte-identical for content/tool/finish chunks. + +The trailing usage chunk is the one pinned envelope difference: v1's wrapper +consumes the wire ``choices: []`` usage chunk into a SYNTHESIZED final chunk +(``stream_chunk_builder`` over its accumulated state, wrapper-cached model +string); the v2 fold passes the wire chunk through verbatim and the future +streaming seam owns that synthesis. The usage test pins both sides of that +contract: byte-identical prefix, equal usage numbers on the tail. +""" + +import copy +import json +import time + +import pytest +from openai.types.chat.chat_completion_chunk import ChatCompletionChunk + +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + +from litellm.translation.engine.stream import fold_events, fold_lines +from litellm.translation.inbound.openai_chat.stream import initial_state +from litellm.translation.providers.openai_compat.stream import parse_event, parse_line +from litellm.translation_seam import to_model_response_stream + +MODEL = "gpt-4o" + + +def _chunk(delta=None, finish=None, usage=None, choices=None): + payload = { + "id": "chatcmpl-S1", + "object": "chat.completion.chunk", + "created": 1718000000, + "model": "gpt-4o-2024-08-06", + "system_fingerprint": "fp_stream", + "service_tier": None, + "choices": [ + { + "index": 0, + "delta": delta or {}, + "logprobs": None, + "finish_reason": finish, + } + ], + "usage": usage, + } + if choices is not None: + payload["choices"] = choices + return payload + + +def _delta(**overrides): + return { + "content": None, + "function_call": None, + "refusal": None, + "role": None, + "tool_calls": None, + **overrides, + } + + +STREAMS = { + "text": [ + _chunk(_delta(role="assistant", content="")), + _chunk(_delta(content="Paris is")), + _chunk(_delta(content=" the capital.")), + _chunk(_delta(), finish="stop"), + ], + "text_no_leading_role": [ + # first content-bearing chunk still gains role: assistant + _chunk(_delta(content="Hi")), + _chunk(_delta(content=" there")), + _chunk(_delta(), finish="stop"), + ], + "tools": [ + _chunk( + _delta( + role="assistant", + tool_calls=[ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ], + ) + ), + _chunk( + _delta( + tool_calls=[ + { + "index": 0, + "id": None, + "type": None, + "function": {"name": None, "arguments": '{"ci'}, + } + ] + ) + ), + _chunk( + _delta( + tool_calls=[ + { + "index": 0, + "id": None, + "type": None, + "function": {"name": None, "arguments": 'ty": "Paris"}'}, + } + ] + ) + ), + _chunk(_delta(), finish="tool_calls"), + ], + "empty_keepalive_swallowed": [ + _chunk(_delta(role="assistant", content="")), + _chunk(_delta()), # empty delta mid-stream: v1 drops it + _chunk(_delta(content="ok")), + _chunk(_delta(), finish="stop"), + ], +} + +_USAGE = { + "completion_tokens": 7, + "prompt_tokens": 11, + "total_tokens": 18, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, +} + +USAGE_STREAM = STREAMS["text"] + [_chunk(choices=[], usage=_USAGE)] + + +def _v1_chunks(events: list, stream_options=None) -> list: + logging = Logging( + model=MODEL, + messages=[{"role": "user", "content": "stream"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="diff-openai-stream", + function_id="diff-openai-stream", + ) + sdk_chunks = ( + ChatCompletionChunk.model_validate(event) for event in copy.deepcopy(events) + ) + wrapper = CustomStreamWrapper( + completion_stream=sdk_chunks, + model=MODEL, + custom_llm_provider="openai", + logging_obj=logging, + stream_options=stream_options, + ) + return [chunk.model_dump() for chunk in wrapper] + + +def _v2_chunks(events: list) -> list: + folded = fold_events( + copy.deepcopy(events), parse_event, initial_state(model=MODEL, dialect="openai") + ) + assert folded.is_ok(), folded.error.summary + return [ + to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump() + for chunk in folded.ok + ] + + +def _norm(chunks: list) -> str: + return json.dumps(chunks, sort_keys=True, default=str) + + +@pytest.mark.parametrize("name", sorted(STREAMS)) +def test_v2_stream_matches_v1(name: str, frozen_ambient) -> None: + events = STREAMS[name] + assert _norm(_v2_chunks(events)) == _norm(_v1_chunks(events)) + + +def test_v2_stream_decodes_sse_lines_identically(frozen_ambient) -> None: + """fold_lines over the raw SSE framing (data: ... / [DONE]) produces the + same chunks as fold_events over the parsed payloads.""" + events = STREAMS["text"] + lines = [f"data: {json.dumps(event)}" for event in events] + ["", "data: [DONE]"] + folded = fold_lines(lines, parse_line, initial_state(model=MODEL, dialect="openai")) + assert folded.is_ok(), folded.error.summary + via_lines = [ + to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump() + for chunk in folded.ok + ] + assert _norm(via_lines) == _norm(_v2_chunks(events)) + + +def test_usage_chunk_passthrough_pins_the_seam_contract(frozen_ambient) -> None: + v1 = _v1_chunks(USAGE_STREAM, stream_options={"include_usage": True}) + v2 = _v2_chunks(USAGE_STREAM) + # content + finish chunks are byte-identical + assert _norm(v2[:-1]) == _norm(v1[: len(v2) - 1]) + # v1's tail is the wrapper-synthesized usage chunk (envelope); v2's tail + # is the wire usage chunk verbatim. The usage numbers must agree. + assert len(v1) == len(v2) + v1_tail, v2_tail = v1[-1], v2[-1] + assert v2_tail["choices"] == [] + assert v1_tail["usage"] is not None and v2_tail["usage"] is not None + for key in ("prompt_tokens", "completion_tokens", "total_tokens"): + assert v1_tail["usage"][key] == v2_tail["usage"][key] == _USAGE[key] + assert ( + v2_tail["usage"]["prompt_tokens_details"]["cached_tokens"] + == _USAGE["prompt_tokens_details"]["cached_tokens"] + ) + assert ( + v2_tail["usage"]["completion_tokens_details"]["reasoning_tokens"] + == _USAGE["completion_tokens_details"]["reasoning_tokens"] + ) + + +_UNSUPPORTED_CHUNKS = { + "function_call_delta": ( + _chunk(_delta(function_call={"name": "f", "arguments": ""})), + "function_call", + ), + "unknown_finish_reason": ( + _chunk(_delta(), finish="function_call"), + "finish_reason", + ), + "multiple_choices": ( + _chunk( + choices=[ + {"index": 0, "delta": _delta(content="a"), "finish_reason": None}, + {"index": 1, "delta": _delta(content="b"), "finish_reason": None}, + ] + ), + "multiple stream choices", + ), + "unknown_delta_key": ( + _chunk({**_delta(content="x"), "reasoning_content": "hmm"}), + "stream delta keys", + ), +} + + +@pytest.mark.parametrize("name", sorted(_UNSUPPORTED_CHUNKS)) +def test_unreachable_chunk_shape_is_a_typed_error(name: str) -> None: + event, reason_fragment = _UNSUPPORTED_CHUNKS[name] + result = parse_event(event) + assert result.is_error(), f"{name} unexpectedly parsed" + assert reason_fragment in result.error.summary, result.error.summary