diff --git a/litellm/translation/CLAUDE.md b/litellm/translation/CLAUDE.md index 7bc1c707ab..6cc415f073 100644 --- a/litellm/translation/CLAUDE.md +++ b/litellm/translation/CLAUDE.md @@ -31,10 +31,15 @@ translation/ │ ├── messages.py # wire messages -> IR messages (hot path) │ ├── parse.py # top-level parse + semantic checks │ ├── response.py # IR ChatResponse -> chat-completion body; carries a -│ │ # ResponseDialect (anthropic | bedrock_converse) -│ │ # because v1's outbound shapes are per-provider +│ │ # ResponseDialect (anthropic | bedrock_converse | +│ │ # gemini) because v1's outbound shapes are +│ │ # per-provider │ └── stream.py # IR stream events -> chunk bodies (pure fold); same -│ # dialect idea via ChunkDialect on StreamState +│ # dialect idea via ChunkDialect on StreamState; +│ # the gemini dialect consumes composite chunk +│ # events (cumulative tool index + the +│ # seen-tool-calls stop->tool_calls rewrite ride +│ # StreamState) ├── providers/ # one subpackage per wire format. Pure, no I/O │ ├── anthropic/ │ │ ├── serialize.py # body assembly in v1 transform_request order @@ -55,13 +60,49 @@ translation/ │ │ ├── response.py # converse JSON -> IR (properties-unwrap json rewrite) │ │ └── stream.py # PARSED converse events -> IR events (pinned at the │ │ # parsed-event seam; AWS framing is botocore's) -│ └── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY: -│ ├── serialize.py # pop model/stream, inject anthropic_version, spoof -│ │ # model for response_format (v1's json-tool forcing) -│ ├── response.py # re-export of anthropic parse_response (invoke -│ │ # response body IS anthropic wire format) -│ └── stream.py # re-export of anthropic parse_event (invoke stream -│ # = anthropic events over AWS framing) +│ ├── bedrock_invoke/ # anthropic serializer + envelope deltas ONLY: +│ │ ├── serialize.py # pop model/stream, inject anthropic_version, spoof +│ │ │ # model for response_format (v1's json-tool forcing; +│ │ │ # RESPONSE_FORMAT_SPOOF_MODEL is shared with vertex) +│ │ ├── response.py # re-export of anthropic parse_response (invoke +│ │ │ # response body IS anthropic wire format) +│ │ └── stream.py # re-export of anthropic parse_event (invoke stream +│ │ # = anthropic events over AWS framing) +│ ├── google_genai/ # ONE generateContent family for BOTH google routes: +│ │ │ # providers "vertex_ai" and "gemini" are the same +│ │ │ # serializer parameterized by the drift list +│ │ │ # (AI Studio refuses https media + forwards +│ │ │ # function-call ids on gemini-3+); auth/host/api- +│ │ │ # version differences are envelope, never here +│ │ ├── serialize.py # cache-marker gate (<1024 chars proves v1 skips the +│ │ │ # context-cache network call), 3-way structured- +│ │ │ # output fork (responseJsonSchema regex vs +│ │ │ # responseSchema+propertyOrdering capability vs +│ │ │ # schema-as-user-message), generationConfig with +│ │ │ # v1's exact snake/camel key mix +│ │ ├── messages.py # contents/system_instruction; tool-result runs flush +│ │ │ # as their own user turns; signature-bearing +│ │ │ # thinking -> thoughtSignature parts +│ │ ├── tools.py # function_declarations + toolConfig (no name munging) +│ │ ├── schema.py # pure ports of v1's _build_vertex_schema pipeline +│ │ │ # ($ref/$defs fail closed) +│ │ ├── params.py # thinkingConfig budget/level fork, finish map, +│ │ │ # gemini-3 default temperature +│ │ ├── response.py # candidates/functionCall/thought parts -> IR; +│ │ │ # usageMetadata modality math (cached/thoughts) +│ │ └── stream.py # parsed GenerateContentResponse events -> the +│ │ # composite `chunk` StreamEvent (complete tool args +│ │ # per chunk; mid-stream error objects are loud) +│ └── vertex_anthropic/ # anthropic serializer + envelope deltas ONLY +│ │ # (the bedrock_invoke pattern): pop model/stream/ +│ │ # json_mode (the anthropic handler pops json_mode +│ │ # BEFORE transform on this path), inject +│ │ # anthropic_version: vertex-2023-10-16, shared +│ │ # json-tool model spoof; betas never emitted for +│ │ # the v2 surface (vertex suppresses them) +│ ├── serialize.py +│ ├── response.py # anthropic parse + request-model restore +│ └── stream.py # re-export of anthropic parse_event └── engine/ ├── pipeline.py # prepare (pure, drives the fallback decision) -> send; │ # per-provider serializer/parser/dialect tables; the @@ -165,7 +206,15 @@ snapshot byte-for-byte (canonical JSON), plus a quirk corpus (4.5 cache ttl, parallel tool config, bedrock name normalization, thinking clamp, tool_choice gates) referenced against v1 in-process. Streams compare v2's fold against the REAL decoders inside `CustomStreamWrapper`, pinned at the parsed-event -seam (AWS event-stream framing is botocore plumbing). Those corpora are the +seam (AWS event-stream framing is botocore plumbing; gemini's SSE line +splitting and accumulated-json fallback are likewise transport in front of +`ModelResponseIterator.chunk_parser`). The google gates +(`test_differential_google_{request,response,stream}.py` over +`characterization_google/`, vendored verbatim from +mateo/translation-characterization-providers) are two-sided the same way, +with the vertex token fetch stubbed at `VertexBase.get_access_token` and a +quirk corpus pinning the 3-way structured-output fork and the +vertex-vs-AI-Studio drift list. Those corpora are the covered surface: a provider's flag turns on only for shapes pinned there. Notable transform-seam facts the corpora pin: v1's anthropic output includes `json_mode: true` for response_format requests, and converse's @@ -178,22 +227,42 @@ A behavior change ships as its own snapshot-diffed PR, never inside a port. ## Current scope -OpenAI-chat-in to three providers out — `anthropic`, `bedrock_converse`, -`bedrock_invoke` — request, response, and stream translation, +OpenAI-chat-in to six providers out — `anthropic`, `bedrock_converse`, +`bedrock_invoke`, `vertex_ai` (gemini route), `gemini` (AI Studio), +`vertex_anthropic` — request, response, and stream translation, differential-green (anthropic: 46-shape corpus + responses + stream replays; -bedrock: the characterization corpus per route + quirk corpus), fail-closed -everywhere else, with non-streaming flag-gated seams live in `completion()`. +bedrock and google: the characterization corpus per route + quirk corpora), +fail-closed everywhere else, with non-streaming flag-gated seams live in +`completion()` (the google forks live in `litellm/translation_seam_google.py` +and route via v1's own `get_vertex_ai_model_route`; their raw body carries +EVERY caller-set OpenAI param so unknown ones fall back typed). Deliberate bedrock fallback surfaces (each names the v1 path): non-Claude bedrock models, native structured outputs (outputConfig), adaptive-effort output_config/beta, response_format+stream (fake_stream), response_format with thinking on invoke (the model spoof crossing), tool history without tools (modify_params dummy tool), empty user text on converse (string-vs-list ambiguity), provisioned `model_id`, `guardrailConfig`, and -``-tagged text in converse responses. Not yet here, each its own -follow-up: streaming seams live; the other inbound schemas -(`anthropic_messages`, `google_genai`, `responses`, `completions`); the other -providers (vertex, azure, `openai_compat`); the same-family fast path -(waits on the opaque-body relay). To add a provider: write -`providers//`, register it in `engine/pipeline._SERIALIZERS` / +``-tagged text in converse responses. +Deliberate google fallback surfaces: cache markers at/above the 1024-char +conservative gate (context-cache create is network I/O; below it v1 provably +skips the call and ignores the markers), cache markers on media blocks, +gs:// and http:// media plus AI-Studio https media (downloads), https media +without a recognizable extension, file/pdf parts (inbound), params outside +the IR (n, seed, penalties, modalities, audio, web_search_options, +service_tier, labels — inbound boundary fallback), hosted tools, +reasoning_effort xhigh/max (v1 raises), gemini-3 thinking budgets (read +`litellm.enable_gemini_default_thinking_level_low`), gemini-3 +reasoning_effort+thinking (v1's conflict check raises), models without +`supports_system_messages` carrying system prompts, multi-candidate / +flagged / promptFeedback-blocked responses, inline media outputs, and +data:-URI tool results (functionResponse.parts). The vertex_anthropic +fallbacks mirror bedrock_invoke (spoof crossing, output_config/format) +plus beta-emitting shapes. Ambient globals force v1 at the seam: +`vertex_ai_safety_settings`, `custom_prompt_dict`, `modify_params`. +Not yet here, each its own follow-up: streaming seams live; the other +inbound schemas (`anthropic_messages`, `google_genai`, `responses`, +`completions`); the other providers (azure, `openai_compat`); the +same-family fast path (waits on the opaque-body relay). To add a provider: +write `providers//`, register it in `engine/pipeline._SERIALIZERS` / `_RESPONSE_PARSERS` / `_RESPONSE_DIALECTS`, add a differential corpus, keep the flag off until differential-green. diff --git a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md index ec821c1b6e..fe2f4a7dcd 100644 --- a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md +++ b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md @@ -1,12 +1,12 @@ -# Translation v2 differential report (anthropic + bedrock) +# Translation v2 differential report (anthropic + bedrock + google) v1 and v2 run over the same corpus; every row must be IDENTICAL (or an explained FALLBACK that v1 serves) for a provider's flag to turn on. -Bedrock rows additionally pin the characterization-corpus snapshot, so -each row proves snapshot == v1-at-HEAD == v2. Regenerate with: +Bedrock and google rows additionally pin the characterization-corpus +snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: `python -m tests.test_litellm.translation.generate_differential_report` -- commit: a10779c756 +- commit: 9603193c02 ## anthropic: request bodies (v1 map_openai_params + transform_request vs v2) @@ -159,4 +159,126 @@ each row proves snapshot == v1-at-HEAD == v2. Regenerate with: - IDENTICAL: text_stream - IDENTICAL: tool_stream +## gemini: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON) + +- IDENTICAL: cache_control_messages +- IDENTICAL: cache_control_tools +- IDENTICAL: full_combo +- IDENTICAL: image_base64 +- SKIPPED (corpus): image_url (v1 downloads URL media for google ai studio transforms (network); the vertex_ai gemini route passes the URL through as fileData and is pinned) +- IDENTICAL: max_completion_tokens +- IDENTICAL: multi_turn +- IDENTICAL: params_sampling +- FALLBACK (v1 serves it): pdf_base64 (file/document parts are outside the v2 inbound surface) +- IDENTICAL: plain_text +- IDENTICAL: reasoning_effort_low +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema +- IDENTICAL: system_prompt +- IDENTICAL: thinking_enabled +- IDENTICAL: thinking_history_blocks +- IDENTICAL: tools_basic +- IDENTICAL: tools_forced_choice +- IDENTICAL: tools_parallel +- IDENTICAL: tools_streamed_args_roundtrip + +## vertex_anthropic: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON) + +- IDENTICAL: cache_control_messages +- IDENTICAL: cache_control_tools +- IDENTICAL: full_combo +- IDENTICAL: image_base64 +- SKIPPED (corpus): image_url (v1 downloads URL media for claude-on-vertex transforms (network); native anthropic passes the URL through and is pinned) +- IDENTICAL: max_completion_tokens +- IDENTICAL: multi_turn +- IDENTICAL: params_sampling +- FALLBACK (v1 serves it): pdf_base64 (file/document parts are outside the v2 inbound surface) +- IDENTICAL: plain_text +- IDENTICAL: reasoning_effort_low +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema +- IDENTICAL: system_prompt +- IDENTICAL: thinking_enabled +- IDENTICAL: thinking_history_blocks +- IDENTICAL: tools_basic +- IDENTICAL: tools_forced_choice +- IDENTICAL: tools_parallel +- IDENTICAL: tools_streamed_args_roundtrip + +## vertex_gemini: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON) + +- IDENTICAL: cache_control_messages +- IDENTICAL: cache_control_tools +- IDENTICAL: full_combo +- IDENTICAL: image_base64 +- IDENTICAL: image_url +- IDENTICAL: max_completion_tokens +- IDENTICAL: multi_turn +- IDENTICAL: params_sampling +- FALLBACK (v1 serves it): pdf_base64 (file/document parts are outside the v2 inbound surface) +- IDENTICAL: plain_text +- IDENTICAL: reasoning_effort_low +- IDENTICAL: response_format_json_object +- IDENTICAL: response_format_json_schema +- IDENTICAL: system_prompt +- IDENTICAL: thinking_enabled +- IDENTICAL: thinking_history_blocks +- IDENTICAL: tools_basic +- IDENTICAL: tools_forced_choice +- IDENTICAL: tools_parallel +- IDENTICAL: tools_streamed_args_roundtrip + +## google quirk corpus (v1 in-process reference) + +- IDENTICAL: quirk gemini3_default_temperature_and_level (vertex_ai/gemini-3-pro-preview) +- IDENTICAL: quirk gemini3_studio_forwards_function_call_ids (gemini/gemini-3-pro-preview) +- IDENTICAL: quirk image_url_format_override (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk multi_system_messages_two_parts (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk parallel_tool_calls_never_reaches_wire (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk reasoning_effort_minimal_model_budget (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk stop_as_string (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk studio_response_schema_property_ordering (gemini/gemini-exp-1206) +- IDENTICAL: quirk studio_schema_prompt_for_unsupported_model (gemini/gemini-1.5-flash) +- IDENTICAL: quirk studio_top_k_passthrough (gemini/gemini-2.5-flash) +- IDENTICAL: quirk system_only_blank_user_message (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk thinking_budget_zero (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk tool_choice_none_mode (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk tool_without_parameters (vertex_ai/gemini-2.5-pro) +- IDENTICAL: quirk vertex_schema_prompt_for_unsupported_capability (vertex_ai/gemini-pro-latest) +- IDENTICAL: quirk vertex_top_k_passthrough (vertex_ai/gemini-2.5-pro) + +## gemini: responses (snapshot == v1 transform_response == v2) + +- IDENTICAL: text_basic +- IDENTICAL: thinking +- IDENTICAL: tool_use + +## vertex_anthropic: responses (snapshot == v1 transform_response == v2) + +- IDENTICAL: text_basic +- IDENTICAL: thinking +- IDENTICAL: tool_use + +## vertex_gemini: responses (snapshot == v1 transform_response == v2) + +- IDENTICAL: text_basic +- IDENTICAL: thinking +- IDENTICAL: tool_use + +## gemini: streams (snapshot == real decoder replay == v2 fold) + +- IDENTICAL: text_stream +- IDENTICAL: tool_stream + +## vertex_anthropic: streams (snapshot == real decoder replay == v2 fold) + +- IDENTICAL: text_stream +- IDENTICAL: tool_stream + +## vertex_gemini: streams (snapshot == real decoder replay == v2 fold) + +- IDENTICAL: text_stream +- IDENTICAL: thinking_stream +- IDENTICAL: tool_stream + Result: 0 divergent rows. Shapes outside the corpus fall back to v1 (fail-closed), so this table is the complete flag-on surface. diff --git a/tests/test_litellm/translation/generate_differential_report.py b/tests/test_litellm/translation/generate_differential_report.py index 13bcbf86ad..87044abaf9 100644 --- a/tests/test_litellm/translation/generate_differential_report.py +++ b/tests/test_litellm/translation/generate_differential_report.py @@ -191,16 +191,168 @@ def _bedrock_stream_rows(lines: list) -> int: return failures +def _stub_vertex_token() -> None: + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + VertexBase.get_access_token = ( # type: ignore[method-assign] + lambda self, credentials, project_id: ( + "char-vertex-token", + project_id or "char-test-project", + ) + ) + + +def _google_request_rows(lines: list) -> int: + from litellm.translation import translate_chat_request + from litellm.translation_seam_google import build_google_deps + + from . import _google_corpus as corpus + from . import test_differential_google_request as req + + failures = 0 + cases = corpus.cases() + for provider_key in sorted(corpus.PROVIDERS): + lines += [ + "", + f"## {provider_key}: request bodies " + "(characterization snapshot == v1-at-HEAD == v2, canonical JSON)", + "", + ] + deps = build_google_deps(corpus.V2_PROVIDERS[provider_key]) + for case_id in sorted(cases): + case = cases[case_id] + if provider_key in case["skip"]: + lines.append( + f"- SKIPPED (corpus): {case_id} ({case['skip'][provider_key]})" + ) + continue + snapshot = ( + corpus.SNAPSHOTS_DIR / "requests" / provider_key / f"{case_id}.json" + ).read_text() + v1_same = ( + corpus.canonical_json( + corpus.run_v1_request_transform(provider_key, case) + ) + == snapshot + ) + result = translate_chat_request( + req._v2_raw(provider_key, case), + corpus.V2_PROVIDERS[provider_key], # type: ignore[arg-type] + deps, + ) + if case_id in req.EXPECTED_FALLBACKS: + ok = result.is_error() and v1_same + failures += 0 if ok else 1 + label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT" + lines.append( + f"- {label}: {case_id} ({req.EXPECTED_FALLBACKS[case_id]})" + ) + continue + v2_same = result.is_ok() and corpus.canonical_json(result.ok) == snapshot + same = v1_same and v2_same + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {case_id}") + lines += ["", "## google quirk corpus (v1 in-process reference)", ""] + import copy as _copy + + for name in sorted(req.QUIRKS): + alias, case, drop_params = req.QUIRKS[name] + v1 = corpus.run_v1_request_transform_for_model( + alias, _copy.deepcopy(case), drop_params=drop_params + ) + model, custom_llm_provider, _ = corpus.resolve_model(alias) + provider_key = {"vertex_ai": "vertex_gemini", "gemini": "gemini"}[ + custom_llm_provider + ] + raw = { + "model": model, + "messages": _copy.deepcopy(case["messages"]), + **_copy.deepcopy(case["params"]), + } + result = req._v2_translate(provider_key, raw, drop_params=drop_params) + same = result.is_ok() and corpus.canonical_json( + result.ok + ) == corpus.canonical_json(v1) + failures += 0 if same else 1 + lines.append( + f"- {'IDENTICAL' if same else 'DIVERGENT'}: quirk {name} ({alias})" + ) + return failures + + +def _google_response_rows(lines: list) -> int: + from . import _google_corpus as corpus + from . import test_differential_google_response as resp + + failures = 0 + for provider_key in sorted(corpus.PROVIDERS): + lines += [ + "", + f"## {provider_key}: responses (snapshot == v1 transform_response == v2)", + "", + ] + for fixture_id in resp._fixture_ids(provider_key): + payload = corpus.load_json( + corpus.FIXTURES_DIR / "responses" / provider_key / f"{fixture_id}.json" + ) + v1 = corpus.run_v1_response_transform( + provider_key, dict(payload), [dict(m) for m in resp._MESSAGES] + ).model_dump() + v2 = resp._v2_model_response(provider_key, payload) + snapshot = corpus.load_json( + corpus.SNAPSHOTS_DIR / "responses" / provider_key / f"{fixture_id}.json" + ) + same = resp._norm(v2) == resp._norm(v1) == resp._norm(snapshot) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {fixture_id}") + return failures + + +def _google_stream_rows(lines: list) -> int: + from . import _google_corpus as corpus + from . import test_differential_google_stream as stream + + failures = 0 + for provider_key in sorted(corpus.PROVIDERS): + lines += [ + "", + f"## {provider_key}: streams (snapshot == real decoder replay == v2 fold)", + "", + ] + for fixture_id in stream._fixture_ids(provider_key): + lines_raw = stream._read_lines(provider_key, fixture_id) + if provider_key == "vertex_anthropic": + v1 = corpus.replay_v1_vertex_anthropic_sse(list(lines_raw)) + v2 = stream._v2_vertex_anthropic_chunks(lines_raw) + normalize = True + else: + v1 = corpus.replay_v1_gemini_sse(provider_key, list(lines_raw)) + v2 = stream._v2_gemini_chunks(provider_key, lines_raw) + normalize = False + snapshot = corpus.load_json( + corpus.SNAPSHOTS_DIR / "streams" / provider_key / f"{fixture_id}.json" + ) + same = ( + stream._norm(v2, normalize) + == stream._norm(v1, normalize) + == stream._norm(snapshot, normalize) + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {fixture_id}") + return failures + + def main() -> None: _freeze_ambient() + _stub_vertex_token() lines = [ - "# Translation v2 differential report (anthropic + bedrock)", + "# Translation v2 differential report (anthropic + bedrock + google)", "", "v1 and v2 run over the same corpus; every row must be IDENTICAL (or an", "explained FALLBACK that v1 serves) for a provider's flag to turn on.", - "Bedrock rows additionally pin the characterization-corpus snapshot, so", - "each row proves snapshot == v1-at-HEAD == v2. Regenerate with:", + "Bedrock and google rows additionally pin the characterization-corpus", + "snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with:", "`python -m tests.test_litellm.translation.generate_differential_report`", "", f"- commit: {_git_sha()}", @@ -210,6 +362,9 @@ def main() -> None: failures += _bedrock_request_rows(lines) failures += _bedrock_response_rows(lines) failures += _bedrock_stream_rows(lines) + failures += _google_request_rows(lines) + failures += _google_response_rows(lines) + failures += _google_stream_rows(lines) lines += [ "", f"Result: {failures} divergent rows."