diff --git a/litellm/translation/CLAUDE.md b/litellm/translation/CLAUDE.md index 7b334c100a..338a5cef58 100644 --- a/litellm/translation/CLAUDE.md +++ b/litellm/translation/CLAUDE.md @@ -83,9 +83,13 @@ translation/ │ │ │ # normalizer; transform_response is dead on the SDK │ │ │ # path); rides the outbound body on ChatResponse.wire │ │ └── stream.py # SSE chunk -> wire_chunk events normalized to the -│ │ # SDK-dump shape; the openai chunk dialect folds -│ │ # them; make_parse_line is the ONE data:-line -│ │ # decode every same-family provider composes +│ │ # SDK-dump shape (service_tier preset to None: +│ │ # the validated SDK chunk materializes it even +│ │ # when the wire omits the key, and v1's wrapper +│ │ # copies it onto every chunk); the openai chunk +│ │ # dialect folds them; make_parse_line is the ONE +│ │ # data:-line decode every same-family provider +│ │ # composes │ ├── google_genai/ # ONE generateContent family for BOTH google routes: │ │ │ # providers "vertex_ai" and "gemini" are the same │ │ │ # serializer parameterized by the drift list @@ -137,8 +141,9 @@ translation/ │ │ ├── serialize.py # azure gates then the openai_compat body verbatim │ │ ├── response.py # re-export of openai parse_response (same live │ │ │ # normalizer; json_mode requests fail closed) -│ │ └── stream.py # openai parser + per-chunk model re-attach and the -│ │ # SDK service_tier default; "azure" chunk dialect +│ │ └── stream.py # openai parser + per-chunk model re-attach; +│ │ # "azure" chunk dialect (the SDK service_tier +│ │ # preset is the shared openai parser's) │ ├── azure_ai/ # the Foundry override set + the Claude route: │ │ ├── guard.py # azure guard + the text-only content-list flatten │ │ ├── serialize.py # azure_ai gates (grok, model-map tool_choice) then diff --git a/litellm/translation/providers/azure/stream.py b/litellm/translation/providers/azure/stream.py index d744a6c2db..ef76dfbac4 100644 --- a/litellm/translation/providers/azure/stream.py +++ b/litellm/translation/providers/azure/stream.py @@ -37,15 +37,10 @@ def parse_event(event: PlainJson) -> _EventResult: chunk = parsed.wire_chunk.value if not isinstance(model, str) or not isinstance(chunk, dict): return base - # The azure decode seam is the VALIDATED SDK ChatCompletionChunk, which - # materializes service_tier=None even when the wire JSON omits it; v1's - # preserve_upstream_non_openai_attributes then copies it onto every - # emitted chunk. - return Ok( - StreamEvent.of_wire_chunk( - JsonBlob(value={"service_tier": None, **chunk, "model": model}) - ) - ) + # The SDK service_tier materialization preset lives in the shared openai + # parser (the SAME validated-ChatCompletionChunk seam); only the model + # re-attach is azure's. + return Ok(StreamEvent.of_wire_chunk(JsonBlob(value={**chunk, "model": model}))) parse_line = make_parse_line(parse_event) diff --git a/litellm/translation/providers/openai_compat/stream.py b/litellm/translation/providers/openai_compat/stream.py index 16ef0d7839..7801275291 100644 --- a/litellm/translation/providers/openai_compat/stream.py +++ b/litellm/translation/providers/openai_compat/stream.py @@ -92,6 +92,13 @@ def parse_event(event: PlainJson) -> _EventResult: normalized_choices = [normalized] identifier = event.get("id") chunk: dict[str, PlainJson] = { + # The decode seam on this path is the VALIDATED SDK + # ChatCompletionChunk, which materializes service_tier=None even when + # the wire JSON omits the key, and v1's wrapper copies it onto every + # emitted chunk; preset it so the real wire shape (key absent) + # matches v1's SDK-materialized null (verifier-wave1a F1). A + # wire-carried value overrides via the spread below. + "service_tier": None, # Keys outside ModelResponseStream's field set ride along verbatim: # v1's wrapper setattrs them onto every emitted chunk # (preserve_upstream_non_openai_attributes, e.g. service_tier). diff --git a/tests/test_litellm/translation/test_differential_openai_stream.py b/tests/test_litellm/translation/test_differential_openai_stream.py index 0e6e1067b4..830fa70d8f 100644 --- a/tests/test_litellm/translation/test_differential_openai_stream.py +++ b/tests/test_litellm/translation/test_differential_openai_stream.py @@ -35,13 +35,17 @@ MODEL = "gpt-4o" def _chunk(delta=None, finish=None, usage=None, choices=None): + # The REAL wire shape: no service_tier key (compat wires essentially + # never carry it). v1's validated SDK ChatCompletionChunk materializes + # service_tier=None anyway; hard-coding the key here biased every replay + # toward that SDK artifact and masked the divergence (verifier-wave1a + # F1) -- the v2 parser now presets the key instead. payload = { "id": "chatcmpl-S1", "object": "chat.completion.chunk", "created": 1718000000, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_stream", - "service_tier": None, "choices": [ { "index": 0, @@ -189,6 +193,21 @@ def test_v2_stream_matches_v1(name: str, frozen_ambient) -> None: assert _norm(_v2_chunks(events)) == _norm(_v1_chunks(events)) +def test_wire_carried_service_tier_overrides_the_preset(frozen_ambient) -> None: + """Both directions of the F1 fix: the de-biased STREAMS rows above pin + key-absent wire chunks (v1's SDK materializes service_tier=None; the v2 + parser presets it), and this row pins that a wire-CARRIED value rides + through the preset verbatim on both sides.""" + events = [ + {**_chunk(_delta(role="assistant", content="hi")), "service_tier": "default"}, + {**_chunk(_delta(), finish="stop"), "service_tier": "default"}, + ] + v1 = _v1_chunks(events) + v2 = _v2_chunks(events) + assert _norm(v2) == _norm(v1) + assert all(chunk["service_tier"] == "default" for chunk in v2) + + def test_v2_stream_decodes_sse_lines_identically(frozen_ambient) -> None: """fold_lines over the raw SSE framing (data: ... / [DONE]) produces the same chunks as fold_events over the parsed payloads."""