mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-24 02:25:29 +00:00
test(translation): openai_compat registry rows, raw-guard table, and two-sided differential gates
Append-only registry rows (serializer/parser/dialect openai) plus a _RAW_GUARDS table the pipeline consults before parse for same-family providers. Request gate: 17 identical rows against v1 in-process (map_openai_params + transform_request) and 17 asserted typed fallbacks. Response gate: 5 identical rows against convert_to_model_response_object (the live SDK-path normalizer) plus loud-error rows for unreachable shapes. Stream gate: byte-identical replays through the real CustomStreamWrapper over SDK chunks, an SSE-line decode equivalence check, and the pinned usage-tail seam contract. DIFFERENTIAL_REPORT.md regenerated: 0 divergent rows.
This commit is contained in:
@@ -55,13 +55,27 @@ translation/
|
||||
│ │ ├── response.py # converse JSON -> IR (properties-unwrap json rewrite)
|
||||
│ │ └── stream.py # PARSED converse events -> IR events (pinned at the
|
||||
│ │ # parsed-event seam; AWS framing is botocore's)
|
||||
│ └── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY:
|
||||
│ ├── serialize.py # pop model/stream, inject anthropic_version, spoof
|
||||
│ │ # model for response_format (v1's json-tool forcing)
|
||||
│ ├── response.py # re-export of anthropic parse_response (invoke
|
||||
│ │ # response body IS anthropic wire format)
|
||||
│ └── stream.py # re-export of anthropic parse_event (invoke stream
|
||||
│ # = anthropic events over AWS framing)
|
||||
│ ├── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY:
|
||||
│ │ ├── serialize.py # pop model/stream, inject anthropic_version, spoof
|
||||
│ │ │ # model for response_format (v1's json-tool forcing)
|
||||
│ │ ├── response.py # re-export of anthropic parse_response (invoke
|
||||
│ │ │ # response body IS anthropic wire format)
|
||||
│ │ └── stream.py # re-export of anthropic parse_event (invoke stream
|
||||
│ │ # = anthropic events over AWS framing)
|
||||
│ └── openai_compat/ # the same-family hub serializer (GPT first consumer)
|
||||
│ ├── guard.py # raw-shape fidelity guard run BEFORE parse: shapes
|
||||
│ │ # the IR cannot round-trip losslessly (string stop,
|
||||
│ │ # message name, image detail, max-tokens key split,
|
||||
│ │ # tool-arg spacing) fall back to v1 as typed errors
|
||||
│ ├── serialize.py # v1's five-touch passthrough body assembly
|
||||
│ ├── messages.py # IR -> openai wire messages (inverse of inbound)
|
||||
│ ├── params.py # o-series/gpt-5 family gates (fail closed until
|
||||
│ │ # their param families are ported), user gate
|
||||
│ ├── response.py # mirrors convert_to_model_response_object (the LIVE
|
||||
│ │ # normalizer; transform_response is dead on the SDK
|
||||
│ │ # path); rides the outbound body on ChatResponse.wire
|
||||
│ └── stream.py # SSE chunk -> wire_chunk events normalized to the
|
||||
│ # SDK-dump shape; the openai chunk dialect folds them
|
||||
└── engine/
|
||||
├── pipeline.py # prepare (pure, drives the fallback decision) -> send;
|
||||
│ # per-provider serializer/parser/dialect tables; the
|
||||
@@ -178,22 +192,36 @@ A behavior change ships as its own snapshot-diffed PR, never inside a port.
|
||||
|
||||
## Current scope
|
||||
|
||||
OpenAI-chat-in to three providers out — `anthropic`, `bedrock_converse`,
|
||||
`bedrock_invoke` — request, response, and stream translation,
|
||||
differential-green (anthropic: 46-shape corpus + responses + stream replays;
|
||||
bedrock: the characterization corpus per route + quirk corpus), fail-closed
|
||||
everywhere else, with non-streaming flag-gated seams live in `completion()`.
|
||||
OpenAI-chat-in to four providers out — `anthropic`, `bedrock_converse`,
|
||||
`bedrock_invoke`, `openai_compat` — request, response, and stream
|
||||
translation, differential-green (anthropic: 46-shape corpus + responses +
|
||||
stream replays; bedrock: the characterization corpus per route + quirk
|
||||
corpus; openai: 17-shape request corpus + 17 typed-fallback rows + response
|
||||
and SDK-chunk stream replays), fail-closed everywhere else, with
|
||||
non-streaming flag-gated seams live in `completion()` for the anthropic and
|
||||
bedrock routes (the openai seam fork is integrator scope and NOT wired).
|
||||
Deliberate bedrock fallback surfaces (each names the v1 path): non-Claude
|
||||
bedrock models, native structured outputs (outputConfig), adaptive-effort
|
||||
output_config/beta, response_format+stream (fake_stream), response_format
|
||||
with thinking on invoke (the model spoof crossing), tool history without
|
||||
tools (modify_params dummy tool), empty user text on converse
|
||||
(string-vs-list ambiguity), provisioned `model_id`, `guardrailConfig`, and
|
||||
`<thinking>`-tagged text in converse responses. Not yet here, each its own
|
||||
follow-up: streaming seams live; the other inbound schemas
|
||||
(`anthropic_messages`, `google_genai`, `responses`, `completions`); the other
|
||||
providers (vertex, azure, `openai_compat`); the same-family fast path
|
||||
(waits on the opaque-body relay). To add a provider: write
|
||||
`<thinking>`-tagged text in converse responses. Deliberate openai fallback
|
||||
surfaces: o-series and gpt-5 model families (their param-rewrite configs are
|
||||
unported), every raw shape the IR cannot round-trip byte-identically (the
|
||||
guard's list: string stop, both max-tokens keys, message `name`, image
|
||||
`detail`/`format`, consecutive same-role turns, single-text content lists,
|
||||
empty tools/stop lists, non-canonical tool-argument JSON spacing, null/
|
||||
non-function tool_calls), the `user` param (model-list gated in v1),
|
||||
`response_format` on gpt-4/gpt-3.5-turbo-16k, `stream_options`, file blocks
|
||||
(v1 downloads http pdf file_ids in-transform), and `http://` image URLs. On
|
||||
streams, the trailing `choices: []` usage chunk passes through verbatim and
|
||||
the wrapper's synthesized final usage chunk stays a seam/envelope concern.
|
||||
Not yet here, each its own follow-up: streaming seams live; the other
|
||||
inbound schemas (`anthropic_messages`, `google_genai`, `responses`,
|
||||
`completions`); the other providers (vertex, azure); the same-family fast
|
||||
path (waits on the opaque-body relay). To add a provider: write
|
||||
`providers/<name>/`, register it in `engine/pipeline._SERIALIZERS` /
|
||||
`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS`, add a differential corpus, keep
|
||||
the flag off until differential-green.
|
||||
`_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS` (plus `_RAW_GUARDS` when the
|
||||
inbound schema is the provider's own family), add a differential corpus,
|
||||
keep the flag off until differential-green.
|
||||
|
||||
@@ -39,6 +39,13 @@ from ..providers.bedrock_invoke import parse_response as bedrock_invoke_parse_re
|
||||
from ..providers.bedrock_invoke import (
|
||||
serialize_request as bedrock_invoke_serialize_request,
|
||||
)
|
||||
from ..providers.openai_compat import parse_response as openai_compat_parse_response
|
||||
from ..providers.openai_compat import (
|
||||
serialize_request as openai_compat_serialize_request,
|
||||
)
|
||||
from ..providers.openai_compat import (
|
||||
unsupported_request_shapes as openai_compat_unsupported_request_shapes,
|
||||
)
|
||||
from .http import Endpoint, ExecuteError, HttpPort, ProviderHttpError
|
||||
|
||||
_Serializer = Callable[[ChatRequest, TranslationDeps], Result[Body, TranslationError]]
|
||||
@@ -51,6 +58,7 @@ _SERIALIZERS: Mapping[Provider, _Serializer] = MappingProxyType(
|
||||
"anthropic": serialize_request,
|
||||
"bedrock_converse": bedrock_converse_serialize_request,
|
||||
"bedrock_invoke": bedrock_invoke_serialize_request,
|
||||
"openai_compat": openai_compat_serialize_request,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -59,6 +67,7 @@ _RESPONSE_PARSERS: Mapping[Provider, _ResponseParser] = MappingProxyType(
|
||||
"anthropic": parse_response,
|
||||
"bedrock_converse": bedrock_converse_parse_response,
|
||||
"bedrock_invoke": bedrock_invoke_parse_response,
|
||||
"openai_compat": openai_compat_parse_response,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -67,9 +76,28 @@ _RESPONSE_DIALECTS: Mapping[Provider, ResponseDialect] = MappingProxyType(
|
||||
"anthropic": "anthropic",
|
||||
"bedrock_converse": "bedrock_converse",
|
||||
"bedrock_invoke": "anthropic", # invoke delegates to the anthropic transform
|
||||
"openai_compat": "openai", # same-family: the wire-derived body
|
||||
}
|
||||
)
|
||||
|
||||
_RawGuard = Callable[[Mapping[str, object]], TranslationError | None]
|
||||
|
||||
_RAW_GUARDS: Mapping[Provider, _RawGuard] = MappingProxyType(
|
||||
# Same-family providers run a raw-shape fidelity guard BEFORE parse: the
|
||||
# inbound parse normalizes wire forms v1 forwards verbatim, so shapes it
|
||||
# cannot round-trip losslessly fall back to v1 as typed errors.
|
||||
{
|
||||
"openai_compat": openai_compat_unsupported_request_shapes,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _raw_guard_error(
|
||||
raw: Mapping[str, object], provider: Provider
|
||||
) -> TranslationError | None:
|
||||
guard = _RAW_GUARDS.get(provider)
|
||||
return guard(raw) if guard is not None else None
|
||||
|
||||
|
||||
def response_dialect(provider: Provider) -> ResponseDialect:
|
||||
return _RESPONSE_DIALECTS.get(provider, "anthropic")
|
||||
@@ -85,6 +113,9 @@ def translate_chat_request(
|
||||
f"provider {provider!r} has no v2 chat serializer yet"
|
||||
)
|
||||
)
|
||||
guard_error = _raw_guard_error(raw, provider)
|
||||
if guard_error is not None:
|
||||
return Error(guard_error)
|
||||
return parse_request(raw).bind(lambda request: serializer(request, deps))
|
||||
|
||||
|
||||
@@ -127,6 +158,9 @@ def prepare_chat_request(
|
||||
f"provider {provider!r} is not fully ported to v2 yet"
|
||||
)
|
||||
)
|
||||
guard_error = _raw_guard_error(raw, provider)
|
||||
if guard_error is not None:
|
||||
return Error(guard_error)
|
||||
match parse_request(raw):
|
||||
case Result(tag="ok", ok=request):
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
from .guard import unsupported_request_shapes
|
||||
from .response import parse_response
|
||||
from .serialize import serialize_request
|
||||
from .stream import parse_event, parse_line
|
||||
|
||||
__all__ = (
|
||||
"parse_event",
|
||||
"parse_line",
|
||||
"parse_response",
|
||||
"serialize_request",
|
||||
"unsupported_request_shapes",
|
||||
)
|
||||
@@ -16,9 +16,10 @@ from __future__ import annotations
|
||||
|
||||
from expression import Error, Ok, Option, Result
|
||||
from expression.collections import Block
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
||||
from ...deps import TranslationDeps
|
||||
from ...errors import TranslationError
|
||||
from ...ir import Body, ChatRequest, PlainJson, ResponseFormat, ToolChoice, ToolDef
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Translation v2 differential report (anthropic + bedrock)
|
||||
# Translation v2 differential report (anthropic + bedrock + openai)
|
||||
|
||||
v1 and v2 run over the same corpus; every row must be IDENTICAL (or an
|
||||
explained FALLBACK that v1 serves) for a provider's flag to turn on.
|
||||
@@ -6,7 +6,7 @@ Bedrock rows additionally pin the characterization-corpus snapshot, so
|
||||
each row proves snapshot == v1-at-HEAD == v2. Regenerate with:
|
||||
`python -m tests.test_litellm.translation.generate_differential_report`
|
||||
|
||||
- commit: a10779c756
|
||||
- commit: c1fee272a0
|
||||
|
||||
## anthropic: request bodies (v1 map_openai_params + transform_request vs v2)
|
||||
|
||||
@@ -76,6 +76,59 @@ each row proves snapshot == v1-at-HEAD == v2. Regenerate with:
|
||||
- IDENTICAL: thinking
|
||||
- IDENTICAL: tools
|
||||
|
||||
## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2)
|
||||
|
||||
- IDENTICAL: cache_control_stripped_everywhere
|
||||
- IDENTICAL: image_base64
|
||||
- IDENTICAL: image_url_string_to_object
|
||||
- IDENTICAL: max_completion_tokens
|
||||
- IDENTICAL: multiturn_stop_list_stream
|
||||
- IDENTICAL: parallel_tool_calls_false
|
||||
- IDENTICAL: response_format_json_object
|
||||
- IDENTICAL: response_format_json_schema_strict
|
||||
- IDENTICAL: system_and_sampling
|
||||
- IDENTICAL: temperature_int_stays_int
|
||||
- IDENTICAL: text
|
||||
- IDENTICAL: tool_call_roundtrip
|
||||
- IDENTICAL: tool_choice_none
|
||||
- IDENTICAL: tool_choice_required
|
||||
- IDENTICAL: tool_choice_specific
|
||||
- IDENTICAL: tools_auto
|
||||
- IDENTICAL: tools_strict
|
||||
- FALLBACK (v1 serves it): both_max_tokens_keys (both max_tokens and max_completion_tokens)
|
||||
- FALLBACK (v1 serves it): consecutive_user_messages (consecutive user messages)
|
||||
- FALLBACK (v1 serves it): empty_tools_list (empty tools list)
|
||||
- FALLBACK (v1 serves it): gpt5_model (OpenAIGPT5Config)
|
||||
- FALLBACK (v1 serves it): http_pdf_file_id (messages)
|
||||
- FALLBACK (v1 serves it): image_detail_key (image_url detail/format)
|
||||
- FALLBACK (v1 serves it): legacy_function_call (function_call)
|
||||
- FALLBACK (v1 serves it): message_name_field (message name field)
|
||||
- FALLBACK (v1 serves it): o_series_model (OpenAIOSeriesConfig)
|
||||
- FALLBACK (v1 serves it): reasoning_effort_plain_gpt (reasoning_effort)
|
||||
- FALLBACK (v1 serves it): response_format_on_gpt4 (outside v1's supported set)
|
||||
- FALLBACK (v1 serves it): single_text_content_list (single-text content list)
|
||||
- FALLBACK (v1 serves it): stop_string_form (string-form stop)
|
||||
- FALLBACK (v1 serves it): stream_options_unsupported (stream_options)
|
||||
- FALLBACK (v1 serves it): tool_call_compact_arguments (non-canonical JSON spacing)
|
||||
- FALLBACK (v1 serves it): top_k_not_openai (top_k)
|
||||
- FALLBACK (v1 serves it): user_param_model_list_gate (open_ai_chat_completion_models)
|
||||
|
||||
## openai_compat: responses (v1 convert_to_model_response_object vs v2)
|
||||
|
||||
- IDENTICAL: cached_and_reasoning_usage_details
|
||||
- IDENTICAL: reasoning_content_key
|
||||
- IDENTICAL: text
|
||||
- IDENTICAL: think_tag_extraction
|
||||
- IDENTICAL: tool_calls_rewrites_stop
|
||||
|
||||
## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold)
|
||||
|
||||
- IDENTICAL: empty_keepalive_swallowed
|
||||
- IDENTICAL: text
|
||||
- IDENTICAL: text_no_leading_role
|
||||
- IDENTICAL: tools
|
||||
- SEAM CONTRACT: usage tail (v2 passes the wire choices=[] usage chunk through; v1's wrapper synthesizes its final usage chunk from it, which is the streaming seam's envelope to reproduce)
|
||||
|
||||
## bedrock_converse: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON)
|
||||
|
||||
- IDENTICAL: cache_control_messages
|
||||
|
||||
@@ -63,6 +63,77 @@ def _anthropic_rows(lines: list) -> int:
|
||||
return failures
|
||||
|
||||
|
||||
def _openai_rows(lines: list) -> int:
|
||||
from litellm.translation import translate_chat_request
|
||||
|
||||
from . import test_differential_openai_request as req
|
||||
from . import test_differential_openai_response as resp
|
||||
from . import test_differential_openai_stream as stream
|
||||
from .conftest import build_real_deps
|
||||
|
||||
failures = 0
|
||||
lines += [
|
||||
"",
|
||||
"## openai_compat: request bodies (v1 map_openai_params + transform_request vs v2)",
|
||||
"",
|
||||
]
|
||||
for name in sorted(req.CORPUS):
|
||||
result = req._v2_body(req.CORPUS[name])
|
||||
same = result.is_ok() and req._norm(result.ok) == req._norm(
|
||||
req._v1_body(req.CORPUS[name])
|
||||
)
|
||||
failures += 0 if same else 1
|
||||
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
|
||||
for name in sorted(req.EXPECTED_FALLBACKS):
|
||||
case, reason = req.EXPECTED_FALLBACKS[name]
|
||||
result = translate_chat_request(dict(case), "openai_compat", build_real_deps())
|
||||
ok = result.is_error() and reason in result.error.summary
|
||||
failures += 0 if ok else 1
|
||||
label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT"
|
||||
lines.append(f"- {label}: {name} ({reason})")
|
||||
lines += [
|
||||
"",
|
||||
"## openai_compat: responses (v1 convert_to_model_response_object vs v2)",
|
||||
"",
|
||||
]
|
||||
for name in sorted(resp._RESPONSES):
|
||||
same = resp._norm(resp._v2_model_response(resp._RESPONSES[name])) == resp._norm(
|
||||
resp._v1_model_response(resp._RESPONSES[name])
|
||||
)
|
||||
failures += 0 if same else 1
|
||||
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
|
||||
lines += [
|
||||
"",
|
||||
"## openai_compat: streams (v1 CustomStreamWrapper over SDK chunks vs v2 fold)",
|
||||
"",
|
||||
]
|
||||
for name in sorted(stream.STREAMS):
|
||||
same = stream._norm(stream._v2_chunks(stream.STREAMS[name])) == stream._norm(
|
||||
stream._v1_chunks(stream.STREAMS[name])
|
||||
)
|
||||
failures += 0 if same else 1
|
||||
lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}")
|
||||
v1 = stream._v1_chunks(stream.USAGE_STREAM, stream_options={"include_usage": True})
|
||||
v2 = stream._v2_chunks(stream.USAGE_STREAM)
|
||||
tail_ok = (
|
||||
len(v1) == len(v2)
|
||||
and stream._norm(v2[:-1]) == stream._norm(v1[: len(v2) - 1])
|
||||
and v2[-1]["choices"] == []
|
||||
and all(
|
||||
v1[-1]["usage"][k] == v2[-1]["usage"][k]
|
||||
for k in ("prompt_tokens", "completion_tokens", "total_tokens")
|
||||
)
|
||||
)
|
||||
failures += 0 if tail_ok else 1
|
||||
lines.append(
|
||||
("- SEAM CONTRACT: " if tail_ok else "- DIVERGENT: ")
|
||||
+ "usage tail (v2 passes the wire choices=[] usage chunk through; v1's"
|
||||
" wrapper synthesizes its final usage chunk from it, which is the"
|
||||
" streaming seam's envelope to reproduce)"
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def _bedrock_request_rows(lines: list) -> int:
|
||||
from litellm.translation import translate_chat_request
|
||||
|
||||
@@ -195,7 +266,7 @@ def main() -> None:
|
||||
_freeze_ambient()
|
||||
|
||||
lines = [
|
||||
"# Translation v2 differential report (anthropic + bedrock)",
|
||||
"# Translation v2 differential report (anthropic + bedrock + openai)",
|
||||
"",
|
||||
"v1 and v2 run over the same corpus; every row must be IDENTICAL (or an",
|
||||
"explained FALLBACK that v1 serves) for a provider's flag to turn on.",
|
||||
@@ -207,6 +278,7 @@ def main() -> None:
|
||||
"",
|
||||
]
|
||||
failures = _anthropic_rows(lines)
|
||||
failures += _openai_rows(lines)
|
||||
failures += _bedrock_request_rows(lines)
|
||||
failures += _bedrock_response_rows(lines)
|
||||
failures += _bedrock_stream_rows(lines)
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Differential parity: v2 IR translation vs the v1 OpenAIGPTConfig chain.
|
||||
|
||||
For each OpenAI chat request in the corpus, the v1 body is produced exactly as
|
||||
``litellm.completion`` would for provider "openai" (``map_openai_params`` then
|
||||
``transform_request``) and compared, as normalized JSON, to the v2 body from
|
||||
``translate_chat_request``. The corpus is the covered surface: the
|
||||
openai_compat v2 flag only turns on for the shapes pinned here. v1 is a
|
||||
near-passthrough (five touches), so byte parity is only possible for shapes
|
||||
the inbound parse round-trips losslessly; everything else must be a TYPED
|
||||
fallback (the raw guard / parse / serializer reasons asserted below), never a
|
||||
silent divergence.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.translation import translate_chat_request
|
||||
|
||||
from .conftest import build_real_deps
|
||||
|
||||
MODEL = "gpt-4o"
|
||||
|
||||
_WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_STRICT_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "report",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"body": {"type": "string"}},
|
||||
"required": ["body"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _assistant_tool_call(call_id, city, name="get_weather"):
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
# json.dumps default spacing: the only argument form the IR
|
||||
# re-dumps byte-identically (the guard rejects the rest)
|
||||
"function": {"name": name, "arguments": json.dumps({"city": city})},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
CORPUS = {
|
||||
"text": {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "Hello, world"}],
|
||||
},
|
||||
"system_and_sampling": {
|
||||
"model": MODEL,
|
||||
"max_tokens": 50,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are helpful"},
|
||||
{"role": "user", "content": "Hi"},
|
||||
],
|
||||
},
|
||||
"multiturn_stop_list_stream": {
|
||||
"model": MODEL,
|
||||
"max_tokens": 64,
|
||||
"stop": ["END", "STOP"],
|
||||
"stream": True,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
"max_completion_tokens": {
|
||||
"model": MODEL,
|
||||
"max_completion_tokens": 128,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
},
|
||||
"temperature_int_stays_int": {
|
||||
"model": MODEL,
|
||||
"temperature": 1,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
},
|
||||
"tools_auto": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"tool_choice": "auto",
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
},
|
||||
"tools_strict": {
|
||||
"model": MODEL,
|
||||
"tools": [_STRICT_TOOL],
|
||||
"messages": [{"role": "user", "content": "report this"}],
|
||||
},
|
||||
"tool_choice_required": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"tool_choice": "required",
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
},
|
||||
"tool_choice_none": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"tool_choice": "none",
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
},
|
||||
"tool_choice_specific": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
},
|
||||
"parallel_tool_calls_false": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"parallel_tool_calls": False,
|
||||
"messages": [{"role": "user", "content": "Weather in Paris and Rome?"}],
|
||||
},
|
||||
"tool_call_roundtrip": {
|
||||
"model": MODEL,
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"messages": [
|
||||
{"role": "user", "content": "Weather in Paris?"},
|
||||
_assistant_tool_call("call_1", "Paris"),
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 20C"},
|
||||
],
|
||||
},
|
||||
"image_url_string_to_object": {
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is this"},
|
||||
{"type": "image_url", "image_url": "https://e.test/a.png"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
"image_base64": {
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "and this"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
"response_format_json_object": {
|
||||
"model": MODEL,
|
||||
"response_format": {"type": "json_object"},
|
||||
"messages": [{"role": "user", "content": "json please"}],
|
||||
},
|
||||
"response_format_json_schema_strict": {
|
||||
"model": MODEL,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"capital": {"type": "string"}},
|
||||
"required": ["capital"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
},
|
||||
},
|
||||
"messages": [{"role": "user", "content": "capital of France?"}],
|
||||
},
|
||||
"cache_control_stripped_everywhere": {
|
||||
# v1 strips cache_control recursively from messages and tools; the IR
|
||||
# carries it as typed metadata and the serializer drops it the same way.
|
||||
"model": MODEL,
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "cached context",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{"type": "text", "text": "question"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Typed fallbacks: each row must return a TranslationError whose summary
|
||||
# carries the reason fragment, so the seam serves the request through v1.
|
||||
# (v1 is NOT invoked for these rows: several would perform I/O, raise, or
|
||||
# depend on get_optional_params interplay outside map_openai_params.)
|
||||
EXPECTED_FALLBACKS = {
|
||||
"o_series_model": (
|
||||
{"model": "o3-mini", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"OpenAIOSeriesConfig",
|
||||
),
|
||||
"gpt5_model": (
|
||||
{"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"OpenAIGPT5Config",
|
||||
),
|
||||
"http_pdf_file_id": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"file_id": "https://e.test/doc.pdf"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
# the inbound schema has no file part at all, so v1's in-transform
|
||||
# pdf download (gpt_transformation.py:236-257) can never be reached
|
||||
"messages",
|
||||
),
|
||||
"stop_string_form": (
|
||||
{"model": MODEL, "stop": "END", "messages": [{"role": "user", "content": "x"}]},
|
||||
"string-form stop",
|
||||
),
|
||||
"both_max_tokens_keys": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"max_tokens": 5,
|
||||
"max_completion_tokens": 6,
|
||||
"messages": [{"role": "user", "content": "x"}],
|
||||
},
|
||||
"both max_tokens and max_completion_tokens",
|
||||
),
|
||||
"message_name_field": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "x", "name": "alice"}],
|
||||
},
|
||||
"message name field",
|
||||
),
|
||||
"consecutive_user_messages": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": "a"},
|
||||
{"role": "user", "content": "b"},
|
||||
],
|
||||
},
|
||||
"consecutive user messages",
|
||||
),
|
||||
"image_detail_key": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "see"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://e.test/a.png",
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
"image_url detail/format",
|
||||
),
|
||||
"tool_call_compact_arguments": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": "w?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
],
|
||||
},
|
||||
"non-canonical JSON spacing",
|
||||
),
|
||||
"single_text_content_list": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "only"}]}
|
||||
],
|
||||
},
|
||||
"single-text content list",
|
||||
),
|
||||
"empty_tools_list": (
|
||||
{"model": MODEL, "tools": [], "messages": [{"role": "user", "content": "x"}]},
|
||||
"empty tools list",
|
||||
),
|
||||
"stream_options_unsupported": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"messages": [{"role": "user", "content": "x"}],
|
||||
},
|
||||
"stream_options",
|
||||
),
|
||||
"user_param_model_list_gate": (
|
||||
{"model": MODEL, "user": "u-1", "messages": [{"role": "user", "content": "x"}]},
|
||||
"open_ai_chat_completion_models",
|
||||
),
|
||||
"top_k_not_openai": (
|
||||
{"model": MODEL, "top_k": 40, "messages": [{"role": "user", "content": "x"}]},
|
||||
"top_k",
|
||||
),
|
||||
"reasoning_effort_plain_gpt": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"reasoning_effort": "high",
|
||||
"messages": [{"role": "user", "content": "x"}],
|
||||
},
|
||||
"reasoning_effort",
|
||||
),
|
||||
"response_format_on_gpt4": (
|
||||
{
|
||||
"model": "gpt-4",
|
||||
"response_format": {"type": "json_object"},
|
||||
"messages": [{"role": "user", "content": "x"}],
|
||||
},
|
||||
"outside v1's supported set",
|
||||
),
|
||||
"legacy_function_call": (
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": "x"},
|
||||
{"role": "assistant", "content": "", "function_call": {"name": "f"}},
|
||||
],
|
||||
},
|
||||
"function_call",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _v1_body(case: dict) -> dict:
|
||||
request = copy.deepcopy(case)
|
||||
config = OpenAIGPTConfig()
|
||||
model = request["model"]
|
||||
params = {
|
||||
key: value for key, value in request.items() if key not in ("model", "messages")
|
||||
}
|
||||
optional = config.map_openai_params(
|
||||
copy.deepcopy(params), {}, model, drop_params=False
|
||||
)
|
||||
return config.transform_request(
|
||||
model, copy.deepcopy(request["messages"]), optional, {}, {}
|
||||
)
|
||||
|
||||
|
||||
def _v2_body(case: dict):
|
||||
return translate_chat_request(
|
||||
copy.deepcopy(case), "openai_compat", build_real_deps()
|
||||
)
|
||||
|
||||
|
||||
def _norm(body: dict) -> str:
|
||||
return json.dumps(body, sort_keys=True, default=str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(CORPUS))
|
||||
def test_v2_request_matches_v1(name: str) -> None:
|
||||
result = _v2_body(CORPUS[name])
|
||||
assert result.is_ok(), result.error.summary
|
||||
assert _norm(result.ok) == _norm(_v1_body(CORPUS[name]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(EXPECTED_FALLBACKS))
|
||||
def test_unsupported_shape_is_a_typed_fallback(name: str) -> None:
|
||||
case, reason_fragment = EXPECTED_FALLBACKS[name]
|
||||
result = _v2_body(case)
|
||||
assert result.is_error(), f"{name} unexpectedly translated: {result.ok!r}"
|
||||
assert reason_fragment in result.error.summary, result.error.summary
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Differential parity for the openai response path.
|
||||
|
||||
The v1 reference is ``convert_to_model_response_object`` over the SDK-dump
|
||||
response shape: the live normalizer on the SDK path (dossier gotcha #1;
|
||||
``OpenAIGPTConfig.transform_response`` never runs there). v2 goes
|
||||
``parse_response`` -> ``serialize_response(dialect="openai")`` ->
|
||||
``to_model_response(usage_style="openai")``, and the two ``ModelResponse``
|
||||
dumps must be identical: wire id/created/system_fingerprint survive, usage is
|
||||
the verbatim ``Usage(**raw)`` passthrough (cached + reasoning details), the
|
||||
finish_reason stop -> tool_calls rewrite fires, and reasoning content is
|
||||
extracted from the key or ``<think>`` tags. Shapes the surface cannot trigger
|
||||
must be typed errors, never silent drops.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
from litellm.translation.inbound.openai_chat import parse_request
|
||||
from litellm.translation.inbound.openai_chat.response import serialize_response
|
||||
from litellm.translation.providers.openai_compat.response import parse_response
|
||||
from litellm.translation_seam import build_translation_deps, to_model_response
|
||||
|
||||
MODEL = "gpt-4o"
|
||||
|
||||
_REQUEST = {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
_USAGE = {
|
||||
"completion_tokens": 6,
|
||||
"prompt_tokens": 12,
|
||||
"total_tokens": 18,
|
||||
"completion_tokens_details": {
|
||||
"accepted_prediction_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"reasoning_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
},
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0},
|
||||
}
|
||||
|
||||
_RESPONSES = {
|
||||
"text": {
|
||||
"id": "chatcmpl-A1",
|
||||
"object": "chat.completion",
|
||||
"created": 1718000000,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"system_fingerprint": "fp_abc123",
|
||||
"service_tier": "default",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": None,
|
||||
"message": {
|
||||
"content": "Hello there.",
|
||||
"role": "assistant",
|
||||
"refusal": None,
|
||||
"annotations": [],
|
||||
"audio": None,
|
||||
"function_call": None,
|
||||
"tool_calls": None,
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
"tool_calls_rewrites_stop": {
|
||||
# finish_reason "stop" + tool_calls -> "tool_calls" on both sides
|
||||
"id": "chatcmpl-T1",
|
||||
"object": "chat.completion",
|
||||
"created": 1718000001,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"system_fingerprint": "fp_x",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": None,
|
||||
"message": {
|
||||
"content": None,
|
||||
"role": "assistant",
|
||||
"refusal": None,
|
||||
"annotations": [],
|
||||
"audio": None,
|
||||
"function_call": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
"cached_and_reasoning_usage_details": {
|
||||
"id": "chatcmpl-U1",
|
||||
"object": "chat.completion",
|
||||
"created": 1718000002,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "length",
|
||||
"logprobs": None,
|
||||
"message": {
|
||||
"content": "partial",
|
||||
"role": "assistant",
|
||||
"refusal": None,
|
||||
"annotations": [],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 100,
|
||||
"prompt_tokens": 1000,
|
||||
"total_tokens": 1100,
|
||||
"completion_tokens_details": {
|
||||
"accepted_prediction_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"reasoning_tokens": 64,
|
||||
"rejected_prediction_tokens": 0,
|
||||
},
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 512},
|
||||
},
|
||||
},
|
||||
"reasoning_content_key": {
|
||||
# compat providers (deepseek-style) return reasoning_content beside
|
||||
# content; v1 lifts it onto Message.reasoning_content
|
||||
"id": "chatcmpl-R1",
|
||||
"object": "chat.completion",
|
||||
"created": 1718000003,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": None,
|
||||
"message": {
|
||||
"content": "answer",
|
||||
"role": "assistant",
|
||||
"reasoning_content": "thought hard",
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12},
|
||||
},
|
||||
"think_tag_extraction": {
|
||||
"id": "chatcmpl-K1",
|
||||
"object": "chat.completion",
|
||||
"created": 1718000004,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": None,
|
||||
"message": {"content": "<think>hmm</think>final", "role": "assistant"},
|
||||
}
|
||||
],
|
||||
"usage": {"completion_tokens": 5, "prompt_tokens": 7, "total_tokens": 12},
|
||||
},
|
||||
}
|
||||
|
||||
_UNSUPPORTED = {
|
||||
"multiple_choices": (
|
||||
{
|
||||
"id": "chatcmpl-N",
|
||||
"created": 1,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{"index": 0, "finish_reason": "stop", "message": {"content": "a"}},
|
||||
{"index": 1, "finish_reason": "stop", "message": {"content": "b"}},
|
||||
],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
"multiple response choices",
|
||||
),
|
||||
"legacy_function_call_output": (
|
||||
{
|
||||
"id": "chatcmpl-F",
|
||||
"created": 1,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "function_call",
|
||||
"message": {
|
||||
"content": None,
|
||||
"role": "assistant",
|
||||
"function_call": {"name": "f", "arguments": "{}"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
"function_call",
|
||||
),
|
||||
"multi_tool_use_parallel_repair": (
|
||||
{
|
||||
"id": "chatcmpl-M",
|
||||
"created": 1,
|
||||
"model": MODEL,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"content": None,
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "multi_tool_use.parallel",
|
||||
"arguments": '{"tool_uses": []}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
"multi_tool_use.parallel",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _v1_model_response(raw: dict) -> dict:
|
||||
result = convert_to_model_response_object(
|
||||
response_object=copy.deepcopy(raw), model_response_object=ModelResponse()
|
||||
)
|
||||
return result.model_dump()
|
||||
|
||||
|
||||
def _v2_model_response(raw: dict) -> dict:
|
||||
parsed = parse_request(copy.deepcopy(_REQUEST))
|
||||
assert parsed.is_ok(), parsed.error.summary
|
||||
response = parse_response(copy.deepcopy(raw), parsed.ok)
|
||||
assert response.is_ok(), response.error.summary
|
||||
body = serialize_response(response.ok, build_translation_deps(), "openai")
|
||||
return to_model_response(body, usage_style="openai").model_dump()
|
||||
|
||||
|
||||
def _norm(payload: dict) -> str:
|
||||
return json.dumps(payload, sort_keys=True, default=str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(_RESPONSES))
|
||||
def test_v2_response_matches_v1(name: str, frozen_ambient) -> None:
|
||||
raw = _RESPONSES[name]
|
||||
assert _norm(_v2_model_response(raw)) == _norm(_v1_model_response(raw))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(_UNSUPPORTED))
|
||||
def test_unreachable_response_shape_is_a_typed_error(name: str) -> None:
|
||||
raw, reason_fragment = _UNSUPPORTED[name]
|
||||
parsed = parse_request(copy.deepcopy(_REQUEST))
|
||||
assert parsed.is_ok(), parsed.error.summary
|
||||
result = parse_response(copy.deepcopy(raw), parsed.ok)
|
||||
assert result.is_error(), f"{name} unexpectedly parsed"
|
||||
assert reason_fragment in result.error.summary, result.error.summary
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Differential parity for openai streaming, pinned at the SDK-chunk seam.
|
||||
|
||||
v1 side: recorded chunk dicts validated into the REAL SDK
|
||||
``ChatCompletionChunk`` models and replayed through ``CustomStreamWrapper``
|
||||
(custom_llm_provider="openai") — the decode path production runs; SSE
|
||||
framing is the OpenAI SDK's plumbing, exactly like AWS framing was
|
||||
botocore's. v2 side: ``engine.stream.fold_events`` with the openai_compat
|
||||
chunk parser and the ``openai`` chunk dialect. Chunk lists must be
|
||||
byte-identical for content/tool/finish chunks.
|
||||
|
||||
The trailing usage chunk is the one pinned envelope difference: v1's wrapper
|
||||
consumes the wire ``choices: []`` usage chunk into a SYNTHESIZED final chunk
|
||||
(``stream_chunk_builder`` over its accumulated state, wrapper-cached model
|
||||
string); the v2 fold passes the wire chunk through verbatim and the future
|
||||
streaming seam owns that synthesis. The usage test pins both sides of that
|
||||
contract: byte-identical prefix, equal usage numbers on the tail.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
from litellm.translation.engine.stream import fold_events, fold_lines
|
||||
from litellm.translation.inbound.openai_chat.stream import initial_state
|
||||
from litellm.translation.providers.openai_compat.stream import parse_event, parse_line
|
||||
from litellm.translation_seam import to_model_response_stream
|
||||
|
||||
MODEL = "gpt-4o"
|
||||
|
||||
|
||||
def _chunk(delta=None, finish=None, usage=None, choices=None):
|
||||
payload = {
|
||||
"id": "chatcmpl-S1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1718000000,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"system_fingerprint": "fp_stream",
|
||||
"service_tier": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": delta or {},
|
||||
"logprobs": None,
|
||||
"finish_reason": finish,
|
||||
}
|
||||
],
|
||||
"usage": usage,
|
||||
}
|
||||
if choices is not None:
|
||||
payload["choices"] = choices
|
||||
return payload
|
||||
|
||||
|
||||
def _delta(**overrides):
|
||||
return {
|
||||
"content": None,
|
||||
"function_call": None,
|
||||
"refusal": None,
|
||||
"role": None,
|
||||
"tool_calls": None,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
STREAMS = {
|
||||
"text": [
|
||||
_chunk(_delta(role="assistant", content="")),
|
||||
_chunk(_delta(content="Paris is")),
|
||||
_chunk(_delta(content=" the capital.")),
|
||||
_chunk(_delta(), finish="stop"),
|
||||
],
|
||||
"text_no_leading_role": [
|
||||
# first content-bearing chunk still gains role: assistant
|
||||
_chunk(_delta(content="Hi")),
|
||||
_chunk(_delta(content=" there")),
|
||||
_chunk(_delta(), finish="stop"),
|
||||
],
|
||||
"tools": [
|
||||
_chunk(
|
||||
_delta(
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": ""},
|
||||
}
|
||||
],
|
||||
)
|
||||
),
|
||||
_chunk(
|
||||
_delta(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": None,
|
||||
"type": None,
|
||||
"function": {"name": None, "arguments": '{"ci'},
|
||||
}
|
||||
]
|
||||
)
|
||||
),
|
||||
_chunk(
|
||||
_delta(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": None,
|
||||
"type": None,
|
||||
"function": {"name": None, "arguments": 'ty": "Paris"}'},
|
||||
}
|
||||
]
|
||||
)
|
||||
),
|
||||
_chunk(_delta(), finish="tool_calls"),
|
||||
],
|
||||
"empty_keepalive_swallowed": [
|
||||
_chunk(_delta(role="assistant", content="")),
|
||||
_chunk(_delta()), # empty delta mid-stream: v1 drops it
|
||||
_chunk(_delta(content="ok")),
|
||||
_chunk(_delta(), finish="stop"),
|
||||
],
|
||||
}
|
||||
|
||||
_USAGE = {
|
||||
"completion_tokens": 7,
|
||||
"prompt_tokens": 11,
|
||||
"total_tokens": 18,
|
||||
"completion_tokens_details": {
|
||||
"accepted_prediction_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"reasoning_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
},
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0},
|
||||
}
|
||||
|
||||
USAGE_STREAM = STREAMS["text"] + [_chunk(choices=[], usage=_USAGE)]
|
||||
|
||||
|
||||
def _v1_chunks(events: list, stream_options=None) -> list:
|
||||
logging = Logging(
|
||||
model=MODEL,
|
||||
messages=[{"role": "user", "content": "stream"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="diff-openai-stream",
|
||||
function_id="diff-openai-stream",
|
||||
)
|
||||
sdk_chunks = (
|
||||
ChatCompletionChunk.model_validate(event) for event in copy.deepcopy(events)
|
||||
)
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=sdk_chunks,
|
||||
model=MODEL,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
stream_options=stream_options,
|
||||
)
|
||||
return [chunk.model_dump() for chunk in wrapper]
|
||||
|
||||
|
||||
def _v2_chunks(events: list) -> list:
|
||||
folded = fold_events(
|
||||
copy.deepcopy(events), parse_event, initial_state(model=MODEL, dialect="openai")
|
||||
)
|
||||
assert folded.is_ok(), folded.error.summary
|
||||
return [
|
||||
to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump()
|
||||
for chunk in folded.ok
|
||||
]
|
||||
|
||||
|
||||
def _norm(chunks: list) -> str:
|
||||
return json.dumps(chunks, sort_keys=True, default=str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(STREAMS))
|
||||
def test_v2_stream_matches_v1(name: str, frozen_ambient) -> None:
|
||||
events = STREAMS[name]
|
||||
assert _norm(_v2_chunks(events)) == _norm(_v1_chunks(events))
|
||||
|
||||
|
||||
def test_v2_stream_decodes_sse_lines_identically(frozen_ambient) -> None:
|
||||
"""fold_lines over the raw SSE framing (data: ... / [DONE]) produces the
|
||||
same chunks as fold_events over the parsed payloads."""
|
||||
events = STREAMS["text"]
|
||||
lines = [f"data: {json.dumps(event)}" for event in events] + ["", "data: [DONE]"]
|
||||
folded = fold_lines(lines, parse_line, initial_state(model=MODEL, dialect="openai"))
|
||||
assert folded.is_ok(), folded.error.summary
|
||||
via_lines = [
|
||||
to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump()
|
||||
for chunk in folded.ok
|
||||
]
|
||||
assert _norm(via_lines) == _norm(_v2_chunks(events))
|
||||
|
||||
|
||||
def test_usage_chunk_passthrough_pins_the_seam_contract(frozen_ambient) -> None:
|
||||
v1 = _v1_chunks(USAGE_STREAM, stream_options={"include_usage": True})
|
||||
v2 = _v2_chunks(USAGE_STREAM)
|
||||
# content + finish chunks are byte-identical
|
||||
assert _norm(v2[:-1]) == _norm(v1[: len(v2) - 1])
|
||||
# v1's tail is the wrapper-synthesized usage chunk (envelope); v2's tail
|
||||
# is the wire usage chunk verbatim. The usage numbers must agree.
|
||||
assert len(v1) == len(v2)
|
||||
v1_tail, v2_tail = v1[-1], v2[-1]
|
||||
assert v2_tail["choices"] == []
|
||||
assert v1_tail["usage"] is not None and v2_tail["usage"] is not None
|
||||
for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
|
||||
assert v1_tail["usage"][key] == v2_tail["usage"][key] == _USAGE[key]
|
||||
assert (
|
||||
v2_tail["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
== _USAGE["prompt_tokens_details"]["cached_tokens"]
|
||||
)
|
||||
assert (
|
||||
v2_tail["usage"]["completion_tokens_details"]["reasoning_tokens"]
|
||||
== _USAGE["completion_tokens_details"]["reasoning_tokens"]
|
||||
)
|
||||
|
||||
|
||||
_UNSUPPORTED_CHUNKS = {
|
||||
"function_call_delta": (
|
||||
_chunk(_delta(function_call={"name": "f", "arguments": ""})),
|
||||
"function_call",
|
||||
),
|
||||
"unknown_finish_reason": (
|
||||
_chunk(_delta(), finish="function_call"),
|
||||
"finish_reason",
|
||||
),
|
||||
"multiple_choices": (
|
||||
_chunk(
|
||||
choices=[
|
||||
{"index": 0, "delta": _delta(content="a"), "finish_reason": None},
|
||||
{"index": 1, "delta": _delta(content="b"), "finish_reason": None},
|
||||
]
|
||||
),
|
||||
"multiple stream choices",
|
||||
),
|
||||
"unknown_delta_key": (
|
||||
_chunk({**_delta(content="x"), "reasoning_content": "hmm"}),
|
||||
"stream delta keys",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(_UNSUPPORTED_CHUNKS))
|
||||
def test_unreachable_chunk_shape_is_a_typed_error(name: str) -> None:
|
||||
event, reason_fragment = _UNSUPPORTED_CHUNKS[name]
|
||||
result = parse_event(event)
|
||||
assert result.is_error(), f"{name} unexpectedly parsed"
|
||||
assert reason_fragment in result.error.summary, result.error.summary
|
||||
Reference in New Issue
Block a user