From ae4643ba8a0807b4c3e7add442460cfce6a56e26 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:44:39 +0000 Subject: [PATCH] test(translation): xai two-sided differential gates over a generated characterization corpus Zero recorded xai fixtures exist anywhere (the characterization branch has none), so the corpus under characterization_xai/ pins v1 IN-PROCESS at HEAD (provenance in its README; regenerate with generate_xai_snapshots) and the gates are two-sided: v1-at-HEAD must equal the committed snapshot (drift) AND v2 must equal it byte-for-byte. Requests run v1 AS EXECUTED (get_optional_params('xai') -> extra_body pop -> transform_request) so the seven R2 raise rows assert v1's UnsupportedParamsError in-process beside v2's typed fallback. Responses run the LIVE XAIChatConfig.transform_response (httpx path) pinning the finish_reason '' chain (stop WITH tool_calls), the reasoning fold + idempotency, websearch billing fields, citations, and the bare-model no-prefix row. Streams replay raw SSE data lines through XAIChatCompletionStreamingHandler + CustomStreamWrapper('xai') (the line seam: the chunk_parser rewrites sit below the parsed-chunk seam) with the usage tail pinned as the inherited openai-port seam contract. DIFFERENTIAL_REPORT.md regenerated: 0 divergent rows. --- .../translation/DIFFERENTIAL_REPORT.md | 73 ++++- tests/test_litellm/translation/_xai_corpus.py | 180 ++++++++++++ .../characterization_xai/README.md | 28 ++ .../cases/cache_control_stripped_grok4.json | 36 +++ .../cases/image_base64_grok4.json | 20 ++ .../cases/image_url_string_grok4.json | 18 ++ .../cases/max_tokens_grok3mini.json | 10 + .../multiturn_stop_list_stream_grok3.json | 23 ++ .../cases/nonuser_name_stripped_grok4.json | 23 ++ .../parallel_tool_calls_false_grok4.json | 30 ++ .../cases/reasoning_effort_grok3mini.json | 10 + .../reasoning_effort_grok_code_fast.json | 10 + .../response_format_json_object_grok4.json | 12 + ..._format_json_schema_strict_kept_grok4.json | 28 ++ .../cases/stop_list_grok2.json | 12 + .../cases/system_and_sampling_grok4.json | 16 ++ .../temperature_int_stays_int_grok4.json | 10 + .../cases/text_grok4.json | 9 + .../tool_call_roundtrip_compact_grok4.json | 48 ++++ .../cases/tool_choice_required_grok4.json | 30 ++ .../cases/tool_choice_specific_grok4.json | 35 +++ .../cases/tools_auto_grok4.json | 30 ++ .../cases/tools_strict_stripped_grok4.json | 30 ++ .../cases/user_param_grok4.json | 10 + .../cached_tokens_usage_passthrough.json | 29 ++ .../finish_empty_string_with_tool_calls.json | 35 +++ .../finish_stop_with_tool_calls_rewrites.json | 35 +++ ...ning_tokens_already_folded_idempotent.json | 28 ++ .../responses/reasoning_tokens_folded.json | 29 ++ .../fixtures/responses/text_basic.json | 26 ++ .../responses/total_tokens_normalized_up.json | 25 ++ .../websearch_sources_and_citations.json | 30 ++ .../citations_dropped_by_the_dict_path.json | 61 ++++ .../streams/empty_keepalive_swallowed.json | 74 +++++ .../fixtures/streams/reasoning_content.json | 76 +++++ .../fixtures/streams/reasoning_renamed.json | 58 ++++ .../fixtures/streams/text.json | 76 +++++ .../streams/text_no_leading_role.json | 57 ++++ .../streams/tools_typeless_continuation.json | 75 +++++ .../streams/usage_tail_include_usage.json | 79 +++++ .../cache_control_stripped_grok4.json | 30 ++ .../requests/image_base64_grok4.json | 20 ++ .../requests/image_url_string_grok4.json | 20 ++ .../requests/max_tokens_grok3mini.json | 10 + .../multiturn_stop_list_stream_grok3.json | 23 ++ .../requests/nonuser_name_stripped_grok4.json | 21 ++ .../parallel_tool_calls_false_grok4.json | 30 ++ .../requests/reasoning_effort_grok3mini.json | 10 + .../reasoning_effort_grok_code_fast.json | 10 + .../response_format_json_object_grok4.json | 12 + ..._format_json_schema_strict_kept_grok4.json | 28 ++ .../snapshots/requests/stop_list_grok2.json | 12 + .../requests/system_and_sampling_grok4.json | 16 ++ .../temperature_int_stays_int_grok4.json | 10 + .../snapshots/requests/text_grok4.json | 9 + .../tool_call_roundtrip_compact_grok4.json | 48 ++++ .../requests/tool_choice_required_grok4.json | 30 ++ .../requests/tool_choice_specific_grok4.json | 35 +++ .../snapshots/requests/tools_auto_grok4.json | 30 ++ .../requests/tools_strict_stripped_grok4.json | 29 ++ .../snapshots/requests/user_param_grok4.json | 10 + .../cached_tokens_usage_passthrough.json | 30 ++ .../finish_empty_string_with_tool_calls.json | 38 +++ .../finish_stop_with_tool_calls_rewrites.json | 36 +++ ...ning_tokens_already_folded_idempotent.json | 29 ++ .../responses/reasoning_tokens_folded.json | 30 ++ .../snapshots/responses/text_basic.json | 27 ++ .../responses/total_tokens_normalized_up.json | 27 ++ .../websearch_sources_and_citations.json | 34 +++ .../citations_dropped_by_the_dict_path.json | 72 +++++ .../streams/empty_keepalive_swallowed.json | 72 +++++ .../snapshots/streams/reasoning_content.json | 98 +++++++ .../snapshots/streams/reasoning_renamed.json | 73 +++++ .../snapshots/streams/text.json | 96 +++++++ .../streams/text_no_leading_role.json | 72 +++++ .../streams/tools_typeless_continuation.json | 92 ++++++ .../streams/usage_tail_include_usage.json | 114 ++++++++ .../generate_differential_report.py | 118 +++++++- .../translation/generate_xai_snapshots.py | 73 +++++ .../test_differential_xai_request.py | 269 ++++++++++++++++++ .../test_differential_xai_response.py | 159 +++++++++++ .../test_differential_xai_stream.py | 173 +++++++++++ 82 files changed, 3696 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/translation/_xai_corpus.py create mode 100644 tests/test_litellm/translation/characterization_xai/README.md create mode 100644 tests/test_litellm/translation/characterization_xai/cases/cache_control_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/image_base64_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/image_url_string_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/max_tokens_grok3mini.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/multiturn_stop_list_stream_grok3.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/nonuser_name_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/parallel_tool_calls_false_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok3mini.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok_code_fast.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/response_format_json_object_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/response_format_json_schema_strict_kept_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/stop_list_grok2.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/system_and_sampling_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/temperature_int_stays_int_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/text_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/tool_call_roundtrip_compact_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/tool_choice_required_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/tool_choice_specific_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/tools_auto_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/tools_strict_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/cases/user_param_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/cached_tokens_usage_passthrough.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_empty_string_with_tool_calls.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_stop_with_tool_calls_rewrites.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_already_folded_idempotent.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_folded.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/text_basic.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/total_tokens_normalized_up.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/responses/websearch_sources_and_citations.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/citations_dropped_by_the_dict_path.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/empty_keepalive_swallowed.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_content.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_renamed.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/text.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/text_no_leading_role.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/tools_typeless_continuation.json create mode 100644 tests/test_litellm/translation/characterization_xai/fixtures/streams/usage_tail_include_usage.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/cache_control_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/image_base64_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/image_url_string_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/max_tokens_grok3mini.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/multiturn_stop_list_stream_grok3.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/nonuser_name_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/parallel_tool_calls_false_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok3mini.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok_code_fast.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_object_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_schema_strict_kept_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/stop_list_grok2.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/system_and_sampling_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/temperature_int_stays_int_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/text_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_call_roundtrip_compact_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_required_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_specific_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_auto_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_strict_stripped_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/requests/user_param_grok4.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/cached_tokens_usage_passthrough.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_empty_string_with_tool_calls.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_stop_with_tool_calls_rewrites.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_already_folded_idempotent.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_folded.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/text_basic.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/total_tokens_normalized_up.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/responses/websearch_sources_and_citations.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/citations_dropped_by_the_dict_path.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/empty_keepalive_swallowed.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_content.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_renamed.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/text.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/text_no_leading_role.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/tools_typeless_continuation.json create mode 100644 tests/test_litellm/translation/characterization_xai/snapshots/streams/usage_tail_include_usage.json create mode 100644 tests/test_litellm/translation/generate_xai_snapshots.py create mode 100644 tests/test_litellm/translation/test_differential_xai_request.py create mode 100644 tests/test_litellm/translation/test_differential_xai_response.py create mode 100644 tests/test_litellm/translation/test_differential_xai_stream.py diff --git a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md index fdd279e363..0905f9bdd1 100644 --- a/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md +++ b/tests/test_litellm/translation/DIFFERENTIAL_REPORT.md @@ -1,4 +1,4 @@ -# Translation v2 differential report (anthropic + bedrock + openai + google + azure) +# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai) v1 and v2 run over the same corpus; every row must be IDENTICAL (or an explained FALLBACK that v1 serves) for a provider's flag to turn on. @@ -6,7 +6,7 @@ Bedrock and google rows additionally pin the characterization-corpus snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: `python -m tests.test_litellm.translation.generate_differential_report` -- commit: ff5c320127 +- commit: 3fe015ef6c ## anthropic: request bodies (v1 map_openai_params + transform_request vs v2) @@ -134,6 +134,75 @@ snapshot, so each row proves snapshot == v1-at-HEAD == v2. Regenerate with: - IDENTICAL: tools - SEAM CONTRACT: usage tail (v2 passes the wire choices=[] usage chunk through; v1's wrapper synthesizes its final usage chunk from it, which is the streaming seam's envelope to reproduce) +## xai: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON; v1 = get_optional_params('xai') + transform_request) + +- IDENTICAL: cache_control_stripped_grok4 +- IDENTICAL: image_base64_grok4 +- IDENTICAL: image_url_string_grok4 +- IDENTICAL: max_tokens_grok3mini +- IDENTICAL: multiturn_stop_list_stream_grok3 +- IDENTICAL: nonuser_name_stripped_grok4 +- IDENTICAL: parallel_tool_calls_false_grok4 +- IDENTICAL: reasoning_effort_grok3mini +- IDENTICAL: reasoning_effort_grok_code_fast +- IDENTICAL: response_format_json_object_grok4 +- IDENTICAL: response_format_json_schema_strict_kept_grok4 +- IDENTICAL: stop_list_grok2 +- IDENTICAL: system_and_sampling_grok4 +- IDENTICAL: temperature_int_stays_int_grok4 +- IDENTICAL: text_grok4 +- IDENTICAL: tool_call_roundtrip_compact_grok4 +- IDENTICAL: tool_choice_required_grok4 +- IDENTICAL: tool_choice_specific_grok4 +- IDENTICAL: tools_auto_grok4 +- IDENTICAL: tools_strict_stripped_grok4 +- IDENTICAL: user_param_grok4 +- FALLBACK (v1 raises UnsupportedParamsError): frequency_penalty_on_grok4 (frequency_penalty) +- FALLBACK (v1 raises UnsupportedParamsError): max_completion_tokens_any_grok (max_completion_tokens) +- FALLBACK (v1 raises UnsupportedParamsError): max_completion_tokens_grok3mini (max_completion_tokens) +- FALLBACK (v1 raises UnsupportedParamsError): reasoning_effort_on_non_reasoning_grok4 (reasoning_effort on non-reasoning xai model) +- FALLBACK (v1 raises UnsupportedParamsError): stop_on_grok3mini (stop on grok-3-mini) +- FALLBACK (v1 raises UnsupportedParamsError): stop_on_grok4 (stop on grok-4-0709) +- FALLBACK (v1 raises UnsupportedParamsError): stop_on_grok_code_fast (stop on grok-code-fast-1) +- FALLBACK (v1 serves it): both_max_tokens_keys (both max_tokens and max_completion_tokens) +- FALLBACK (v1 serves it): consecutive_user_messages (consecutive user messages) +- FALLBACK (v1 serves it): empty_tools_list (empty tools list) +- FALLBACK (v1 serves it): explicit_stream_false_reaches_wire (explicit stream: false) +- FALLBACK (v1 serves it): frequency_penalty_supported_family_outside_ir (frequency_penalty) +- FALLBACK (v1 serves it): image_detail_key (image_url detail/format) +- FALLBACK (v1 serves it): logprobs_outside_ir (logprobs) +- FALLBACK (v1 serves it): nested_tool_strict_below_function (nested 'strict' key) +- FALLBACK (v1 serves it): presence_penalty_outside_ir (presence_penalty) +- FALLBACK (v1 serves it): seed_outside_ir (seed) +- FALLBACK (v1 serves it): stream_options_outside_ir (stream_options) +- FALLBACK (v1 serves it): string_form_stop_supported_family (string-form stop) +- FALLBACK (v1 serves it): top_k_not_an_xai_param (top_k) +- FALLBACK (v1 serves it): use_xai_oauth_pkce_flow (PKCE) +- FALLBACK (v1 serves it): user_message_name_forwarded_by_v1 (message name field) +- FALLBACK (v1 serves it): web_search_options_responses_bridge (Responses-API bridge) + +## xai: responses (snapshot == v1 XAIChatConfig.transform_response == v2; the LIVE httpx-path normalizer incl. the usage post-steps) + +- IDENTICAL: cached_tokens_usage_passthrough +- IDENTICAL: finish_empty_string_with_tool_calls +- IDENTICAL: finish_stop_with_tool_calls_rewrites +- IDENTICAL: reasoning_tokens_already_folded_idempotent +- IDENTICAL: reasoning_tokens_folded +- IDENTICAL: text_basic +- IDENTICAL: total_tokens_normalized_up +- IDENTICAL: websearch_sources_and_citations + +## xai: streams (snapshot == v1 line-seam replay through XAIChatCompletionStreamingHandler + CustomStreamWrapper('xai') == v2 xai dialect) + +- IDENTICAL: citations_dropped_by_the_dict_path +- IDENTICAL: empty_keepalive_swallowed +- IDENTICAL: reasoning_content +- IDENTICAL: reasoning_renamed +- IDENTICAL: text +- IDENTICAL: text_no_leading_role +- IDENTICAL: tools_typeless_continuation +- SEAM CONTRACT: usage_tail_include_usage (v1's chunk_parser injects a dummy choice so the wrapper swallows the tail and synthesizes the final usage chunk; v2 passes the wire choices=[] chunk through with the FOLDED usage for the streaming seam to synthesize from) + ## azure: request bodies (v1 api-version-aware map_openai_params + transform_request vs v2) - IDENTICAL: deployment_with_base_model diff --git a/tests/test_litellm/translation/_xai_corpus.py b/tests/test_litellm/translation/_xai_corpus.py new file mode 100644 index 0000000000..a05b5044fa --- /dev/null +++ b/tests/test_litellm/translation/_xai_corpus.py @@ -0,0 +1,180 @@ +"""Shared plumbing for the xai (Grok) differential gates. + +The reference corpus under ``characterization_xai/`` is GENERATED, not +vendored: the characterization branch +(mateo/translation-characterization-providers) carries zero xai fixtures, so +every snapshot here pins v1 IN-PROCESS AT HEAD (the primary reference per +the differential rule) invoked exactly the way the xai httpx handler runs +(provenance documented in characterization_xai/README.md; regenerate with +``python -m tests.test_litellm.translation.generate_xai_snapshots``). + +The v1 invokers mirror main.py's dedicated xai elif (main.py:2289 -> +``base_llm_http_handler.completion``): + +- requests: ``get_optional_params(custom_llm_provider="xai")`` (the + RAISE-unless-drop_params gate over XAIChatConfig's supported list) with + completion()'s ``stream=None`` default, then the handler's ``extra_body`` + pop (hh:398-399; the injected ``{}`` merges nothing onto the wire), then + ``XAIChatConfig.transform_request``. +- responses: ``XAIChatConfig.transform_response`` over an ``httpx.Response`` + — LIVE on the httpx path (the inverse of the openai SDK route), including + the websearch/fold/normalize usage post-steps. +- streams: SSE ``data:`` lines through ``XAIChatCompletionStreamingHandler`` + + ``CustomStreamWrapper(custom_llm_provider="xai")`` (the line seam the + dossier prescribes: the chunk_parser rewrites are xai BEHAVIOR and sit + below the parsed-chunk seam). +""" + +import copy +import json +import pathlib +import time +from typing import Any, Dict, List, Optional + +import httpx + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.xai.chat.transformation import ( + XAIChatCompletionStreamingHandler, + XAIChatConfig, +) +from litellm.types.utils import ModelResponse +from litellm.utils import get_optional_params + +CORPUS_DIR = pathlib.Path(__file__).parent / "characterization_xai" +CASES_DIR = CORPUS_DIR / "cases" +FIXTURES_DIR = CORPUS_DIR / "fixtures" +SNAPSHOTS_DIR = CORPUS_DIR / "snapshots" + +STREAM_MODEL = "grok-3-mini" + +FROZEN_TIME = 1718064000.0 # matches the translation conftest frozen_ambient + + +def load_json(path: pathlib.Path) -> Any: + with open(path) as f: + return json.load(f) + + +def corpus(kind: str) -> Dict[str, Any]: + directory = CASES_DIR if kind == "cases" else FIXTURES_DIR / kind + return {path.stem: load_json(path) for path in sorted(directory.glob("*.json"))} + + +def jsonable(obj: Any) -> Any: + if hasattr(obj, "model_dump"): + return jsonable(obj.model_dump()) + if isinstance(obj, dict): + return {str(k): jsonable(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [jsonable(v) for v in obj] + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + return repr(obj) + + +def canonical_json(obj: Any) -> str: + return json.dumps(jsonable(obj), indent=2, sort_keys=True) + "\n" + + +def run_v1_request_transform(case: Dict[str, Any]) -> Dict[str, Any]: + """May RAISE UnsupportedParamsError: that IS the pinned v1 behavior for + the R2 gate rows (the differential asserts the raise, never a remap).""" + request = copy.deepcopy(case) + model = request.pop("model") + messages = request.pop("messages") + optional_params = get_optional_params( + model=model, + custom_llm_provider="xai", + messages=copy.deepcopy(messages), + stream=request.pop("stream", None), + **request, + ) + optional_params.pop("extra_body", None) + return XAIChatConfig().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + +def make_logging(model: str, messages: List[dict], stream: bool = False) -> Logging: + logging_obj = Logging( + model=model, + messages=messages, + stream=stream, + call_type="completion", + start_time=time.time(), + litellm_call_id="diff-xai-call-id", + function_id="diff-xai-function-id", + ) + logging_obj.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={} + ) + return logging_obj + + +def run_v1_response_transform( + provider_response: Dict[str, Any], model: str +) -> ModelResponse: + messages = [{"role": "user", "content": "hi"}] + raw_response = httpx.Response( + status_code=200, + json=copy.deepcopy(provider_response), + request=httpx.Request("POST", "https://api.x.ai/v1/chat/completions"), + ) + return XAIChatConfig().transform_response( + model=model, + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=make_logging(model, messages), + request_data={}, + messages=messages, + optional_params={}, + litellm_params={}, + encoding=litellm.encoding, + api_key=None, + json_mode=None, + ) + + +def replay_xai_sse_lines( + events: List[dict], stream_options: Optional[dict] = None +) -> List[dict]: + lines = [f"data: {json.dumps(event)}" for event in copy.deepcopy(events)] + lines.append("data: [DONE]") + handler = XAIChatCompletionStreamingHandler( + streaming_response=iter(lines), sync_stream=True + ) + wrapper = CustomStreamWrapper( + completion_stream=handler, + model=STREAM_MODEL, + custom_llm_provider="xai", + logging_obj=make_logging( + STREAM_MODEL, [{"role": "user", "content": "stream"}], stream=True + ), + stream_options=stream_options, + ) + return [chunk.model_dump() for chunk in wrapper] + + +__all__ = ( + "CASES_DIR", + "CORPUS_DIR", + "FIXTURES_DIR", + "FROZEN_TIME", + "SNAPSHOTS_DIR", + "STREAM_MODEL", + "canonical_json", + "corpus", + "jsonable", + "load_json", + "make_logging", + "replay_xai_sse_lines", + "run_v1_request_transform", + "run_v1_response_transform", +) diff --git a/tests/test_litellm/translation/characterization_xai/README.md b/tests/test_litellm/translation/characterization_xai/README.md new file mode 100644 index 0000000000..ac31c8c6a6 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/README.md @@ -0,0 +1,28 @@ +# characterization_xai — Grok (xai) corpus + +Provenance: GENERATED from v1 in-process at HEAD, not vendored. The +characterization branch (mateo/translation-characterization-providers) has +zero xai fixtures and no recorded xai vendor traffic exists in the repo, so +every snapshot pins v1-as-executed (the primary reference under the +differential rule in 05-provider-expansion.md). The wire shapes in +`fixtures/` are hand-authored to the documented xAI API shapes plus the +quirks researcher-3 verified in-process (finish_reason "", reasoning token +accounting, num_sources_used, the `choices: []` usage tail). + +- `cases/` — OpenAI-format chat requests (request seam input). Models cover + grok-2-1212 / grok-3 / grok-3-mini / grok-4-0709 / grok-code-fast-1, the + serve side of all three per-model gates (stop, frequency_penalty, + reasoning_effort). +- `fixtures/responses/` — provider response bodies `{model, body}`. +- `fixtures/streams/` — SSE chunk payload lists `{stream_options, events}`. +- `snapshots/` — v1 output at the pinned seams (canonical JSON: sorted keys, + 2-space indent, trailing newline): + - `requests/`: `get_optional_params("xai")` -> extra_body pop -> + `XAIChatConfig.transform_request` (the httpx-handler call order) + - `responses/`: `XAIChatConfig.transform_response().model_dump()` (LIVE on + the httpx path; includes the xai usage post-steps) + - `streams/`: SSE data-lines -> `XAIChatCompletionStreamingHandler` -> + `CustomStreamWrapper("xai")` chunk dumps, ambient frozen at 1718064000 + +Regenerate (a reviewed snapshot diff, never silent): +`LITELLM_LOCAL_MODEL_COST_MAP=True python -m tests.test_litellm.translation.generate_xai_snapshots` diff --git a/tests/test_litellm/translation/characterization_xai/cases/cache_control_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/cache_control_stripped_grok4.json new file mode 100644 index 0000000000..11e2ac7f00 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/cache_control_stripped_grok4.json @@ -0,0 +1,36 @@ +{ + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "cached context", + "type": "text" + }, + { + "text": "question", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "cache_control": { + "type": "ephemeral" + }, + "name": "lookup", + "parameters": { + "properties": {}, + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/image_base64_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/image_base64_grok4.json new file mode 100644 index 0000000000..803e2e314b --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/image_base64_grok4.json @@ -0,0 +1,20 @@ +{ + "messages": [ + { + "content": [ + { + "text": "and this", + "type": "text" + }, + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/image_url_string_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/image_url_string_grok4.json new file mode 100644 index 0000000000..91fb7297b9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/image_url_string_grok4.json @@ -0,0 +1,18 @@ +{ + "messages": [ + { + "content": [ + { + "text": "what is this", + "type": "text" + }, + { + "image_url": "https://e.test/a.png", + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/max_tokens_grok3mini.json b/tests/test_litellm/translation/characterization_xai/cases/max_tokens_grok3mini.json new file mode 100644 index 0000000000..6c46219051 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/max_tokens_grok3mini.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 64, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "grok-3-mini" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/multiturn_stop_list_stream_grok3.json b/tests/test_litellm/translation/characterization_xai/cases/multiturn_stop_list_stream_grok3.json new file mode 100644 index 0000000000..e67a4ecc96 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/multiturn_stop_list_stream_grok3.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 64, + "messages": [ + { + "content": "Hello", + "role": "user" + }, + { + "content": "Hi there", + "role": "assistant" + }, + { + "content": "How are you?", + "role": "user" + } + ], + "model": "grok-3", + "stop": [ + "END", + "STOP" + ], + "stream": true +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/nonuser_name_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/nonuser_name_stripped_grok4.json new file mode 100644 index 0000000000..019204b5a4 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/nonuser_name_stripped_grok4.json @@ -0,0 +1,23 @@ +{ + "messages": [ + { + "content": "s", + "name": "sys", + "role": "system" + }, + { + "content": "q", + "role": "user" + }, + { + "content": "a", + "name": "bot", + "role": "assistant" + }, + { + "content": "q2", + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/parallel_tool_calls_false_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/parallel_tool_calls_false_grok4.json new file mode 100644 index 0000000000..054461b735 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/parallel_tool_calls_false_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris and Rome?", + "role": "user" + } + ], + "model": "grok-4-0709", + "parallel_tool_calls": false, + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok3mini.json b/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok3mini.json new file mode 100644 index 0000000000..08e5804275 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok3mini.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "think", + "role": "user" + } + ], + "model": "grok-3-mini", + "reasoning_effort": "high" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok_code_fast.json b/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok_code_fast.json new file mode 100644 index 0000000000..d9c6802530 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/reasoning_effort_grok_code_fast.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "code", + "role": "user" + } + ], + "model": "grok-code-fast-1", + "reasoning_effort": "low" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/response_format_json_object_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/response_format_json_object_grok4.json new file mode 100644 index 0000000000..c7fc2ed3d6 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/response_format_json_object_grok4.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "content": "json please", + "role": "user" + } + ], + "model": "grok-4-0709", + "response_format": { + "type": "json_object" + } +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/response_format_json_schema_strict_kept_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/response_format_json_schema_strict_kept_grok4.json new file mode 100644 index 0000000000..9b231251c9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/response_format_json_schema_strict_kept_grok4.json @@ -0,0 +1,28 @@ +{ + "messages": [ + { + "content": "capital of France?", + "role": "user" + } + ], + "model": "grok-4-0709", + "response_format": { + "json_schema": { + "name": "answer", + "schema": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + } +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/stop_list_grok2.json b/tests/test_litellm/translation/characterization_xai/cases/stop_list_grok2.json new file mode 100644 index 0000000000..36c2b1a787 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/stop_list_grok2.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "content": "x", + "role": "user" + } + ], + "model": "grok-2-1212", + "stop": [ + "END" + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/system_and_sampling_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/system_and_sampling_grok4.json new file mode 100644 index 0000000000..c5c6c3a420 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/system_and_sampling_grok4.json @@ -0,0 +1,16 @@ +{ + "max_tokens": 50, + "messages": [ + { + "content": "You are helpful", + "role": "system" + }, + { + "content": "Hi", + "role": "user" + } + ], + "model": "grok-4-0709", + "temperature": 0.5, + "top_p": 0.9 +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/temperature_int_stays_int_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/temperature_int_stays_int_grok4.json new file mode 100644 index 0000000000..bed9d52414 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/temperature_int_stays_int_grok4.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "grok-4-0709", + "temperature": 1 +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/text_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/text_grok4.json new file mode 100644 index 0000000000..370c873c46 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/text_grok4.json @@ -0,0 +1,9 @@ +{ + "messages": [ + { + "content": "Hello, world", + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/tool_call_roundtrip_compact_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/tool_call_roundtrip_compact_grok4.json new file mode 100644 index 0000000000..b476841f0d --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/tool_call_roundtrip_compact_grok4.json @@ -0,0 +1,48 @@ +{ + "messages": [ + { + "content": "w?", + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_weather" + }, + "id": "call_1", + "type": "function" + } + ] + }, + { + "content": "Sunny, 20C", + "role": "tool", + "tool_call_id": "call_1" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/tool_choice_required_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/tool_choice_required_grok4.json new file mode 100644 index 0000000000..d32692e6c9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/tool_choice_required_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": "required", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/tool_choice_specific_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/tool_choice_specific_grok4.json new file mode 100644 index 0000000000..1985219aee --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/tool_choice_specific_grok4.json @@ -0,0 +1,35 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/tools_auto_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/tools_auto_grok4.json new file mode 100644 index 0000000000..c97f826db5 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/tools_auto_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": "auto", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/tools_strict_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/tools_strict_stripped_grok4.json new file mode 100644 index 0000000000..1bb1d16e42 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/tools_strict_stripped_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "report this", + "role": "user" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "name": "report", + "parameters": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + } + }, + "required": [ + "body" + ], + "type": "object" + }, + "strict": true + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/cases/user_param_grok4.json b/tests/test_litellm/translation/characterization_xai/cases/user_param_grok4.json new file mode 100644 index 0000000000..be4dbeda73 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/cases/user_param_grok4.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "x", + "role": "user" + } + ], + "model": "grok-4-0709", + "user": "u-1" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/cached_tokens_usage_passthrough.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/cached_tokens_usage_passthrough.json new file mode 100644 index 0000000000..3a6f5f9a9e --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/cached_tokens_usage_passthrough.json @@ -0,0 +1,29 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "content": "partial", + "role": "assistant" + } + } + ], + "created": 1718000007, + "id": "resp-c1", + "model": "grok-4-0709", + "object": "chat.completion", + "usage": { + "completion_tokens": 100, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 512 + }, + "total_tokens": 1100 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_empty_string_with_tool_calls.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_empty_string_with_tool_calls.json new file mode 100644 index 0000000000..b2c0f43ac9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_empty_string_with_tool_calls.json @@ -0,0 +1,35 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "", + "index": 0, + "logprobs": null, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_weather" + }, + "id": "call_1", + "type": "function" + } + ] + } + } + ], + "created": 1718000001, + "id": "resp-r1", + "model": "grok-4-0709", + "object": "chat.completion", + "usage": { + "completion_tokens": 6, + "prompt_tokens": 12, + "total_tokens": 18 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_stop_with_tool_calls_rewrites.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_stop_with_tool_calls_rewrites.json new file mode 100644 index 0000000000..a7f6bed90c --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/finish_stop_with_tool_calls_rewrites.json @@ -0,0 +1,35 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Rome\"}", + "name": "get_weather" + }, + "id": "call_2", + "type": "function" + } + ] + } + } + ], + "created": 1718000002, + "id": "resp-r2", + "model": "grok-4-0709", + "object": "chat.completion", + "usage": { + "completion_tokens": 6, + "prompt_tokens": 12, + "total_tokens": 18 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_already_folded_idempotent.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_already_folded_idempotent.json new file mode 100644 index 0000000000..1176ac0693 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_already_folded_idempotent.json @@ -0,0 +1,28 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "a", + "role": "assistant" + } + } + ], + "created": 1718000004, + "id": "resp-f2", + "model": "grok-3-mini", + "object": "chat.completion", + "usage": { + "completion_tokens": 12, + "completion_tokens_details": { + "reasoning_tokens": 7 + }, + "prompt_tokens": 10, + "total_tokens": 22 + } + }, + "model": "grok-3-mini" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_folded.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_folded.json new file mode 100644 index 0000000000..54e86dcf59 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/reasoning_tokens_folded.json @@ -0,0 +1,29 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "answer", + "reasoning_content": "thought hard", + "role": "assistant" + } + } + ], + "created": 1718000003, + "id": "resp-f1", + "model": "grok-3-mini", + "object": "chat.completion", + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "reasoning_tokens": 7 + }, + "prompt_tokens": 10, + "total_tokens": 22 + } + }, + "model": "grok-3-mini" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/text_basic.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/text_basic.json new file mode 100644 index 0000000000..7d16a1bd73 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/text_basic.json @@ -0,0 +1,26 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "Hello there.", + "role": "assistant" + } + } + ], + "created": 1718000000, + "id": "resp-t1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": "fp_xai_1", + "usage": { + "completion_tokens": 6, + "prompt_tokens": 12, + "total_tokens": 18 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/total_tokens_normalized_up.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/total_tokens_normalized_up.json new file mode 100644 index 0000000000..16899330f2 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/total_tokens_normalized_up.json @@ -0,0 +1,25 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "a", + "role": "assistant" + } + } + ], + "created": 1718000006, + "id": "resp-n1", + "model": "grok-4-0709", + "object": "chat.completion", + "usage": { + "completion_tokens": 9, + "prompt_tokens": 30, + "total_tokens": 5 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/responses/websearch_sources_and_citations.json b/tests/test_litellm/translation/characterization_xai/fixtures/responses/websearch_sources_and_citations.json new file mode 100644 index 0000000000..d421d9e824 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/responses/websearch_sources_and_citations.json @@ -0,0 +1,30 @@ +{ + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "cited answer", + "role": "assistant" + } + } + ], + "citations": [ + "https://a.test", + "https://b.test" + ], + "created": 1718000005, + "id": "resp-w1", + "model": "grok-4-0709", + "object": "chat.completion", + "usage": { + "completion_tokens": 9, + "num_sources_used": 3, + "prompt_tokens": 30, + "total_tokens": 39 + } + }, + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/citations_dropped_by_the_dict_path.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/citations_dropped_by_the_dict_path.json new file mode 100644 index 0000000000..c2b724c9ce --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/citations_dropped_by_the_dict_path.json @@ -0,0 +1,61 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "content": "c", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "ited" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": [ + "https://a.test" + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/empty_keepalive_swallowed.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/empty_keepalive_swallowed.json new file mode 100644 index 0000000000..b61b6a7513 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/empty_keepalive_swallowed.json @@ -0,0 +1,74 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "content": "", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "ok" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_content.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_content.json new file mode 100644 index 0000000000..6d6b371843 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_content.json @@ -0,0 +1,76 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "reasoning_content": "Let me think", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "reasoning_content": " more" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "Answer" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_renamed.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_renamed.json new file mode 100644 index 0000000000..7f9b46ac5e --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/reasoning_renamed.json @@ -0,0 +1,58 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "reasoning": "hmm", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "A" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/text.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/text.json new file mode 100644 index 0000000000..9b5948542b --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/text.json @@ -0,0 +1,76 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "content": "", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "Paris is" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": " the capital." + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/text_no_leading_role.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/text_no_leading_role.json new file mode 100644 index 0000000000..7d825891dd --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/text_no_leading_role.json @@ -0,0 +1,57 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "content": "Hi" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": " there" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/tools_typeless_continuation.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/tools_typeless_continuation.json new file mode 100644 index 0000000000..4ece297856 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/tools_typeless_continuation.json @@ -0,0 +1,75 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "get_weather" + }, + "id": "call_1", + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}" + }, + "index": 0 + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + } + ], + "stream_options": null +} diff --git a/tests/test_litellm/translation/characterization_xai/fixtures/streams/usage_tail_include_usage.json b/tests/test_litellm/translation/characterization_xai/fixtures/streams/usage_tail_include_usage.json new file mode 100644 index 0000000000..85a99b0612 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/fixtures/streams/usage_tail_include_usage.json @@ -0,0 +1,79 @@ +{ + "events": [ + { + "choices": [ + { + "delta": { + "content": "", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": { + "content": "Hello" + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": null + }, + { + "choices": [], + "created": 1718000000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_xai_s", + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "reasoning_tokens": 7 + }, + "prompt_tokens": 171, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "total_tokens": 180 + } + } + ], + "stream_options": { + "include_usage": true + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/cache_control_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/cache_control_stripped_grok4.json new file mode 100644 index 0000000000..f271e5f190 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/cache_control_stripped_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": [ + { + "text": "cached context", + "type": "text" + }, + { + "text": "question", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "name": "lookup", + "parameters": { + "properties": {}, + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_base64_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_base64_grok4.json new file mode 100644 index 0000000000..803e2e314b --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_base64_grok4.json @@ -0,0 +1,20 @@ +{ + "messages": [ + { + "content": [ + { + "text": "and this", + "type": "text" + }, + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_url_string_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_url_string_grok4.json new file mode 100644 index 0000000000..9c3ef40350 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/image_url_string_grok4.json @@ -0,0 +1,20 @@ +{ + "messages": [ + { + "content": [ + { + "text": "what is this", + "type": "text" + }, + { + "image_url": { + "url": "https://e.test/a.png" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/max_tokens_grok3mini.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/max_tokens_grok3mini.json new file mode 100644 index 0000000000..6c46219051 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/max_tokens_grok3mini.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 64, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "grok-3-mini" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/multiturn_stop_list_stream_grok3.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/multiturn_stop_list_stream_grok3.json new file mode 100644 index 0000000000..e67a4ecc96 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/multiturn_stop_list_stream_grok3.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 64, + "messages": [ + { + "content": "Hello", + "role": "user" + }, + { + "content": "Hi there", + "role": "assistant" + }, + { + "content": "How are you?", + "role": "user" + } + ], + "model": "grok-3", + "stop": [ + "END", + "STOP" + ], + "stream": true +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/nonuser_name_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/nonuser_name_stripped_grok4.json new file mode 100644 index 0000000000..4b73fc32d6 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/nonuser_name_stripped_grok4.json @@ -0,0 +1,21 @@ +{ + "messages": [ + { + "content": "s", + "role": "system" + }, + { + "content": "q", + "role": "user" + }, + { + "content": "a", + "role": "assistant" + }, + { + "content": "q2", + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/parallel_tool_calls_false_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/parallel_tool_calls_false_grok4.json new file mode 100644 index 0000000000..054461b735 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/parallel_tool_calls_false_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris and Rome?", + "role": "user" + } + ], + "model": "grok-4-0709", + "parallel_tool_calls": false, + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok3mini.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok3mini.json new file mode 100644 index 0000000000..08e5804275 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok3mini.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "think", + "role": "user" + } + ], + "model": "grok-3-mini", + "reasoning_effort": "high" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok_code_fast.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok_code_fast.json new file mode 100644 index 0000000000..d9c6802530 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/reasoning_effort_grok_code_fast.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "code", + "role": "user" + } + ], + "model": "grok-code-fast-1", + "reasoning_effort": "low" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_object_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_object_grok4.json new file mode 100644 index 0000000000..c7fc2ed3d6 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_object_grok4.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "content": "json please", + "role": "user" + } + ], + "model": "grok-4-0709", + "response_format": { + "type": "json_object" + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_schema_strict_kept_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_schema_strict_kept_grok4.json new file mode 100644 index 0000000000..9b231251c9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/response_format_json_schema_strict_kept_grok4.json @@ -0,0 +1,28 @@ +{ + "messages": [ + { + "content": "capital of France?", + "role": "user" + } + ], + "model": "grok-4-0709", + "response_format": { + "json_schema": { + "name": "answer", + "schema": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/stop_list_grok2.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/stop_list_grok2.json new file mode 100644 index 0000000000..36c2b1a787 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/stop_list_grok2.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "content": "x", + "role": "user" + } + ], + "model": "grok-2-1212", + "stop": [ + "END" + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/system_and_sampling_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/system_and_sampling_grok4.json new file mode 100644 index 0000000000..c5c6c3a420 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/system_and_sampling_grok4.json @@ -0,0 +1,16 @@ +{ + "max_tokens": 50, + "messages": [ + { + "content": "You are helpful", + "role": "system" + }, + { + "content": "Hi", + "role": "user" + } + ], + "model": "grok-4-0709", + "temperature": 0.5, + "top_p": 0.9 +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/temperature_int_stays_int_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/temperature_int_stays_int_grok4.json new file mode 100644 index 0000000000..bed9d52414 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/temperature_int_stays_int_grok4.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "grok-4-0709", + "temperature": 1 +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/text_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/text_grok4.json new file mode 100644 index 0000000000..370c873c46 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/text_grok4.json @@ -0,0 +1,9 @@ +{ + "messages": [ + { + "content": "Hello, world", + "role": "user" + } + ], + "model": "grok-4-0709" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_call_roundtrip_compact_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_call_roundtrip_compact_grok4.json new file mode 100644 index 0000000000..b476841f0d --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_call_roundtrip_compact_grok4.json @@ -0,0 +1,48 @@ +{ + "messages": [ + { + "content": "w?", + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_weather" + }, + "id": "call_1", + "type": "function" + } + ] + }, + { + "content": "Sunny, 20C", + "role": "tool", + "tool_call_id": "call_1" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_required_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_required_grok4.json new file mode 100644 index 0000000000..d32692e6c9 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_required_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": "required", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_specific_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_specific_grok4.json new file mode 100644 index 0000000000..1985219aee --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tool_choice_specific_grok4.json @@ -0,0 +1,35 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_auto_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_auto_grok4.json new file mode 100644 index 0000000000..c97f826db5 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_auto_grok4.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "grok-4-0709", + "tool_choice": "auto", + "tools": [ + { + "function": { + "description": "Get weather", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_strict_stripped_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_strict_stripped_grok4.json new file mode 100644 index 0000000000..f6b836f16d --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/tools_strict_stripped_grok4.json @@ -0,0 +1,29 @@ +{ + "messages": [ + { + "content": "report this", + "role": "user" + } + ], + "model": "grok-4-0709", + "tools": [ + { + "function": { + "name": "report", + "parameters": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + } + }, + "required": [ + "body" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/requests/user_param_grok4.json b/tests/test_litellm/translation/characterization_xai/snapshots/requests/user_param_grok4.json new file mode 100644 index 0000000000..be4dbeda73 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/requests/user_param_grok4.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "x", + "role": "user" + } + ], + "model": "grok-4-0709", + "user": "u-1" +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/cached_tokens_usage_passthrough.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/cached_tokens_usage_passthrough.json new file mode 100644 index 0000000000..39c6c249ec --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/cached_tokens_usage_passthrough.json @@ -0,0 +1,30 @@ +{ + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "partial", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "created": 1718000007, + "id": "resp-c1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 100, + "completion_tokens_details": null, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 512 + }, + "total_tokens": 1100 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_empty_string_with_tool_calls.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_empty_string_with_tool_calls.json new file mode 100644 index 0000000000..ae8b73a449 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_empty_string_with_tool_calls.json @@ -0,0 +1,38 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": null, + "function_call": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_weather" + }, + "id": "call_1", + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "" + } + } + ], + "created": 1718000001, + "id": "resp-r1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 6, + "completion_tokens_details": null, + "prompt_tokens": 12, + "prompt_tokens_details": null, + "total_tokens": 18 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_stop_with_tool_calls_rewrites.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_stop_with_tool_calls_rewrites.json new file mode 100644 index 0000000000..55da9d96b1 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/finish_stop_with_tool_calls_rewrites.json @@ -0,0 +1,36 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "function_call": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Rome\"}", + "name": "get_weather" + }, + "id": "call_2", + "type": "function" + } + ] + }, + "provider_specific_fields": {} + } + ], + "created": 1718000002, + "id": "resp-r2", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 6, + "completion_tokens_details": null, + "prompt_tokens": 12, + "prompt_tokens_details": null, + "total_tokens": 18 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_already_folded_idempotent.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_already_folded_idempotent.json new file mode 100644 index 0000000000..25c3d0a5ca --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_already_folded_idempotent.json @@ -0,0 +1,29 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "a", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "created": 1718000004, + "id": "resp-f2", + "model": "grok-3-mini", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 12, + "completion_tokens_details": { + "reasoning_tokens": 7 + }, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 22 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_folded.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_folded.json new file mode 100644 index 0000000000..3622fcfd85 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/reasoning_tokens_folded.json @@ -0,0 +1,30 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "answer", + "function_call": null, + "reasoning_content": "thought hard", + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "created": 1718000003, + "id": "resp-f1", + "model": "grok-3-mini", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 12, + "completion_tokens_details": { + "reasoning_tokens": 7 + }, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 22 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/text_basic.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/text_basic.json new file mode 100644 index 0000000000..4da03984ab --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/text_basic.json @@ -0,0 +1,27 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello there.", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "created": 1718000000, + "id": "resp-t1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": "fp_xai_1", + "usage": { + "completion_tokens": 6, + "completion_tokens_details": null, + "prompt_tokens": 12, + "prompt_tokens_details": null, + "total_tokens": 18 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/total_tokens_normalized_up.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/total_tokens_normalized_up.json new file mode 100644 index 0000000000..d1251b1748 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/total_tokens_normalized_up.json @@ -0,0 +1,27 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "a", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "created": 1718000006, + "id": "resp-n1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 9, + "completion_tokens_details": null, + "prompt_tokens": 30, + "prompt_tokens_details": null, + "total_tokens": 39 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/responses/websearch_sources_and_citations.json b/tests/test_litellm/translation/characterization_xai/snapshots/responses/websearch_sources_and_citations.json new file mode 100644 index 0000000000..c49c2b4784 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/responses/websearch_sources_and_citations.json @@ -0,0 +1,34 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "cited answer", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "provider_specific_fields": {} + } + ], + "citations": [ + "https://a.test", + "https://b.test" + ], + "created": 1718000005, + "id": "resp-w1", + "model": "grok-4-0709", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 9, + "completion_tokens_details": null, + "num_sources_used": 3, + "prompt_tokens": 30, + "prompt_tokens_details": { + "web_search_requests": 3 + }, + "total_tokens": 39 + } +} diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/citations_dropped_by_the_dict_path.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/citations_dropped_by_the_dict_path.json new file mode 100644 index 0000000000..f97b1d45b3 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/citations_dropped_by_the_dict_path.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "c", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "ited", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/empty_keepalive_swallowed.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/empty_keepalive_swallowed.json new file mode 100644 index 0000000000..4895db6381 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/empty_keepalive_swallowed.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "ok", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_content.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_content.json new file mode 100644 index 0000000000..950af4bccd --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_content.json @@ -0,0 +1,98 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "provider_specific_fields": null, + "reasoning_content": "Let me think", + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "provider_specific_fields": null, + "reasoning_content": " more", + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Answer", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_renamed.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_renamed.json new file mode 100644 index 0000000000..b4f742a5ca --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/reasoning_renamed.json @@ -0,0 +1,73 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "provider_specific_fields": null, + "reasoning_content": "hmm", + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "A", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/text.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/text.json new file mode 100644 index 0000000000..291139830c --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/text.json @@ -0,0 +1,96 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris is", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": " the capital.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/text_no_leading_role.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/text_no_leading_role.json new file mode 100644 index 0000000000..47d849ae11 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/text_no_leading_role.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Hi", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": " there", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/tools_typeless_continuation.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/tools_typeless_continuation.json new file mode 100644 index 0000000000..67083990de --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/tools_typeless_continuation.json @@ -0,0 +1,92 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "get_weather" + }, + "id": "call_1", + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/test_litellm/translation/characterization_xai/snapshots/streams/usage_tail_include_usage.json b/tests/test_litellm/translation/characterization_xai/snapshots/streams/usage_tail_include_usage.json new file mode 100644 index 0000000000..7ae1da2b81 --- /dev/null +++ b/tests/test_litellm/translation/characterization_xai/snapshots/streams/usage_tail_include_usage.json @@ -0,0 +1,114 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Hello", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "cmpl-xs1", + "model": "grok-3-mini", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "usage": { + "completion_tokens": 9, + "completion_tokens_details": { + "accepted_prediction_tokens": null, + "audio_tokens": null, + "image_tokens": null, + "reasoning_tokens": 7, + "rejected_prediction_tokens": null, + "text_tokens": null, + "video_tokens": null + }, + "prompt_tokens": 171, + "prompt_tokens_details": { + "audio_tokens": null, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": null, + "video_tokens": null + }, + "total_tokens": 180 + } + } +] diff --git a/tests/test_litellm/translation/generate_differential_report.py b/tests/test_litellm/translation/generate_differential_report.py index 2eaeb44c8c..275296fa0f 100644 --- a/tests/test_litellm/translation/generate_differential_report.py +++ b/tests/test_litellm/translation/generate_differential_report.py @@ -146,6 +146,121 @@ def _openai_rows(lines: list) -> int: return failures +def _xai_rows(lines: list) -> int: + from litellm.exceptions import UnsupportedParamsError + + from . import _xai_corpus as corpus + from . import test_differential_xai_request as req + from . import test_differential_xai_response as resp + from . import test_differential_xai_stream as stream + + failures = 0 + lines += [ + "", + "## xai: request bodies (characterization snapshot == v1-at-HEAD == v2, canonical JSON; v1 = get_optional_params('xai') + transform_request)", + "", + ] + for name in sorted(req.CASES): + case = req.CASES[name] + snapshot = (corpus.SNAPSHOTS_DIR / "requests" / f"{name}.json").read_text() + v1_same = corpus.canonical_json(corpus.run_v1_request_transform(case)) == ( + snapshot + ) + result = req._v2(case) + v2_same = result.is_ok() and req._norm(result.ok) == req._norm( + corpus.load_json(corpus.SNAPSHOTS_DIR / "requests" / f"{name}.json") + ) + same = v1_same and v2_same + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + for name in sorted(req.V1_RAISES): + case, reason = req.V1_RAISES[name] + result = req._v2(case) + try: + corpus.run_v1_request_transform(case) + raised = False + except UnsupportedParamsError: + raised = True + ok = result.is_error() and reason in result.error.summary and raised + failures += 0 if ok else 1 + label = "FALLBACK (v1 raises UnsupportedParamsError)" if ok else "DIVERGENT" + lines.append(f"- {label}: {name} ({reason})") + for name in sorted(req.EXPECTED_FALLBACKS): + case, reason = req.EXPECTED_FALLBACKS[name] + result = req._v2(case) + ok = result.is_error() and reason in result.error.summary + failures += 0 if ok else 1 + label = "FALLBACK (v1 serves it)" if ok else "DIVERGENT" + lines.append(f"- {label}: {name} ({reason})") + lines += [ + "", + "## xai: responses (snapshot == v1 XAIChatConfig.transform_response == v2; the LIVE httpx-path normalizer incl. the usage post-steps)", + "", + ] + responses = corpus.corpus("responses") + for name in sorted(responses): + row = responses[name] + snapshot = (corpus.SNAPSHOTS_DIR / "responses" / f"{name}.json").read_text() + same = ( + corpus.canonical_json( + corpus.run_v1_response_transform(row["body"], row["model"]) + ) + == snapshot + and corpus.canonical_json( + resp._v2_model_response(row["body"], row["model"]) + ) + == snapshot + ) + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + lines += [ + "", + "## xai: streams (snapshot == v1 line-seam replay through XAIChatCompletionStreamingHandler + CustomStreamWrapper('xai') == v2 xai dialect)", + "", + ] + streams = corpus.corpus("streams") + for name in sorted(streams): + row = streams[name] + snapshot_text = (corpus.SNAPSHOTS_DIR / "streams" / f"{name}.json").read_text() + v1_same = ( + corpus.canonical_json( + corpus.replay_xai_sse_lines(row["events"], row["stream_options"]) + ) + == snapshot_text + ) + if name == stream._TAIL_ROW: + snapshot = corpus.load_json( + corpus.SNAPSHOTS_DIR / "streams" / f"{name}.json" + ) + v2 = stream._v2_chunks(row["events"]) + tail_ok = ( + v1_same + and len(v2) == len(snapshot) + and stream._norm(v2[:-1]) == stream._norm(snapshot[: len(v2) - 1]) + and v2[-1]["choices"] == [] + and all( + snapshot[-1]["usage"][k] == v2[-1]["usage"][k] + for k in ("prompt_tokens", "completion_tokens", "total_tokens") + ) + ) + failures += 0 if tail_ok else 1 + lines.append( + ("- SEAM CONTRACT: " if tail_ok else "- DIVERGENT: ") + + f"{name} (v1's chunk_parser injects a dummy choice so the" + " wrapper swallows the tail and synthesizes the final usage" + " chunk; v2 passes the wire choices=[] chunk through with the" + " FOLDED usage for the streaming seam to synthesize from)" + ) + continue + v2_same = stream._norm(stream._v2_chunks(row["events"])) == stream._norm( + corpus.load_json(corpus.SNAPSHOTS_DIR / "streams" / f"{name}.json") + ) + same = v1_same and v2_same + failures += 0 if same else 1 + lines.append(f"- {'IDENTICAL' if same else 'DIVERGENT'}: {name}") + return failures + + def _azure_rows(lines: list) -> int: import os @@ -617,7 +732,7 @@ def main() -> None: _stub_vertex_token() lines = [ - "# Translation v2 differential report (anthropic + bedrock + openai + google + azure)", + "# Translation v2 differential report (anthropic + bedrock + openai + google + azure + xai)", "", "v1 and v2 run over the same corpus; every row must be IDENTICAL (or an", "explained FALLBACK that v1 serves) for a provider's flag to turn on.", @@ -630,6 +745,7 @@ def main() -> None: ] failures = _anthropic_rows(lines) failures += _openai_rows(lines) + failures += _xai_rows(lines) failures += _azure_rows(lines) failures += _azure_ai_rows(lines) failures += _bedrock_request_rows(lines) diff --git a/tests/test_litellm/translation/generate_xai_snapshots.py b/tests/test_litellm/translation/generate_xai_snapshots.py new file mode 100644 index 0000000000..615b4eaee7 --- /dev/null +++ b/tests/test_litellm/translation/generate_xai_snapshots.py @@ -0,0 +1,73 @@ +"""Regenerate the characterization_xai snapshots from v1 IN-PROCESS at HEAD. + +Run: LITELLM_LOCAL_MODEL_COST_MAP=True \ + python -m tests.test_litellm.translation.generate_xai_snapshots + +Provenance: there are no recorded xai vendor fixtures anywhere (the +characterization branch has zero), so the snapshots pin v1-as-executed — +the primary differential reference. The drift gate in the xai differential +tests re-runs the same invokers and fails if v1 at HEAD ever stops matching +the committed snapshots; regenerating is a reviewed snapshot-diff, never a +silent step. + +Ambient freeze mirrors tests' ``frozen_ambient``: stream chunks stamp +``created`` from ``time.time`` and the wrapper mints fastuuid ids. +""" + +import itertools +import pathlib +import sys +import time +import uuid + + +def _freeze_ambient() -> None: + import fastuuid + + import litellm._uuid + + counter = itertools.count(1) + + def fake_uuid4(): + return uuid.UUID(int=next(counter)) + + uuid.uuid4 = fake_uuid4 # type: ignore[assignment] + fastuuid.uuid4 = fake_uuid4 + litellm._uuid.uuid4 = fake_uuid4 + time.time = lambda: FROZEN_TIME # type: ignore[assignment] + + +def _write(path: pathlib.Path, payload: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(payload) + print(f"wrote {path}") + + +if __name__ == "__main__": + sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[3])) + from tests.test_litellm.translation._xai_corpus import ( + FROZEN_TIME, + SNAPSHOTS_DIR, + canonical_json, + corpus, + run_v1_request_transform, + run_v1_response_transform, + replay_xai_sse_lines, + ) + + _freeze_ambient() + for name, case in corpus("cases").items(): + _write( + SNAPSHOTS_DIR / "requests" / f"{name}.json", + canonical_json(run_v1_request_transform(case)), + ) + for name, row in corpus("responses").items(): + _write( + SNAPSHOTS_DIR / "responses" / f"{name}.json", + canonical_json(run_v1_response_transform(row["body"], row["model"])), + ) + for name, row in corpus("streams").items(): + _write( + SNAPSHOTS_DIR / "streams" / f"{name}.json", + canonical_json(replay_xai_sse_lines(row["events"], row["stream_options"])), + ) diff --git a/tests/test_litellm/translation/test_differential_xai_request.py b/tests/test_litellm/translation/test_differential_xai_request.py new file mode 100644 index 0000000000..0a9130ac91 --- /dev/null +++ b/tests/test_litellm/translation/test_differential_xai_request.py @@ -0,0 +1,269 @@ +"""Differential parity: v2 xai translation vs the v1 XAIChatConfig chain. + +Two-sided over the generated characterization corpus (provenance: +characterization_xai/README.md): v1 AT HEAD must still equal the committed +snapshot (drift guard) AND v2 must equal the snapshot byte-for-byte. The v1 +invoker is ``get_optional_params(custom_llm_provider="xai")`` -> extra_body +pop -> ``transform_request`` — v1 AS EXECUTED on the httpx path, never bare +``map_openai_params`` (whose max_completion_tokens rename arm is dead code: +``_check_valid_arg`` raises first; researcher-3 R2). + +The R2 gate rows therefore pin the RAISE: v1 must raise +UnsupportedParamsError in-process AND v2 must return a typed fallback, so +flag-on traffic gets v1's own error, never a remap. +""" + +import copy +import json + +import pytest + +from litellm.exceptions import UnsupportedParamsError + +from litellm.translation import translate_chat_request + +from ._xai_corpus import ( + SNAPSHOTS_DIR, + canonical_json, + corpus, + load_json, + run_v1_request_transform, +) +from .conftest import build_real_deps + +CASES = corpus("cases") + +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} + +_USER = [{"role": "user", "content": "x"}] + +# Rows where v1 RAISES UnsupportedParamsError (the R2 supported-list gate); +# v2 must be a typed fallback so v1 serves its own raise. The reason fragment +# is asserted on the v2 error; the raise is asserted on v1 in-process. +V1_RAISES = { + "max_completion_tokens_any_grok": ( + {"model": "grok-4-0709", "max_completion_tokens": 128, "messages": _USER}, + "max_completion_tokens", + ), + "max_completion_tokens_grok3mini": ( + {"model": "grok-3-mini", "max_completion_tokens": 128, "messages": _USER}, + "max_completion_tokens", + ), + "stop_on_grok4": ( + {"model": "grok-4-0709", "stop": ["END"], "messages": _USER}, + "stop on grok-4-0709", + ), + "stop_on_grok3mini": ( + {"model": "grok-3-mini", "stop": ["END"], "messages": _USER}, + "stop on grok-3-mini", + ), + "stop_on_grok_code_fast": ( + {"model": "grok-code-fast-1", "stop": ["END"], "messages": _USER}, + "stop on grok-code-fast-1", + ), + "reasoning_effort_on_non_reasoning_grok4": ( + {"model": "grok-4-0709", "reasoning_effort": "high", "messages": _USER}, + "reasoning_effort on non-reasoning xai model", + ), + "frequency_penalty_on_grok4": ( + # parse-level fallback (penalties are outside the IR); v1's gate + # raises for this family, so the fallback serves v1's own error + {"model": "grok-4-0709", "frequency_penalty": 0.5, "messages": _USER}, + "frequency_penalty", + ), +} + +# Typed fallbacks where v1 SERVES the request (v1 is not invoked: the seam +# routes these to v1 untouched, so v1's behavior is by-construction v1's). +EXPECTED_FALLBACKS = { + "web_search_options_responses_bridge": ( + { + "model": "grok-4-0709", + "web_search_options": {"search_context_size": "high"}, + "messages": _USER, + }, + "Responses-API bridge", + ), + "use_xai_oauth_pkce_flow": ( + {"model": "grok-4-0709", "use_xai_oauth": True, "messages": _USER}, + "PKCE", + ), + "explicit_stream_false_reaches_wire": ( + {"model": "grok-4-0709", "stream": False, "messages": _USER}, + "explicit stream: false", + ), + "user_message_name_forwarded_by_v1": ( + { + "model": "grok-4-0709", + "messages": [{"role": "user", "content": "x", "name": "alice"}], + }, + "message name field", + ), + "nested_tool_strict_below_function": ( + { + "model": "grok-4-0709", + "tools": [ + { + "type": "function", + "function": { + "name": "f", + "parameters": { + "type": "object", + "properties": {"strict": {"type": "boolean"}}, + }, + }, + } + ], + "messages": _USER, + }, + "nested 'strict' key", + ), + "presence_penalty_outside_ir": ( + # v1 passes presence_penalty through for every grok model; the hub + # keeps penalties parse-level fallbacks (integrator's unified + # _raw_openai_body semantics), so v1 serves it + {"model": "grok-4-0709", "presence_penalty": 0.5, "messages": _USER}, + "presence_penalty", + ), + "frequency_penalty_supported_family_outside_ir": ( + # supported on grok-3-mini in v1; still a parse-level fallback + {"model": "grok-3-mini", "frequency_penalty": 0.5, "messages": _USER}, + "frequency_penalty", + ), + "seed_outside_ir": ( + {"model": "grok-4-0709", "seed": 42, "messages": _USER}, + "seed", + ), + "logprobs_outside_ir": ( + {"model": "grok-4-0709", "logprobs": True, "messages": _USER}, + "logprobs", + ), + "stream_options_outside_ir": ( + { + "model": "grok-4-0709", + "stream": True, + "stream_options": {"include_usage": True}, + "messages": _USER, + }, + "stream_options", + ), + "string_form_stop_supported_family": ( + {"model": "grok-3", "stop": "END", "messages": _USER}, + "string-form stop", + ), + "both_max_tokens_keys": ( + { + "model": "grok-4-0709", + "max_tokens": 5, + "max_completion_tokens": 6, + "messages": _USER, + }, + "both max_tokens and max_completion_tokens", + ), + "top_k_not_an_xai_param": ( + {"model": "grok-4-0709", "top_k": 40, "messages": _USER}, + "top_k", + ), + "image_detail_key": ( + { + "model": "grok-4-0709", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "see"}, + { + "type": "image_url", + "image_url": { + "url": "https://e.test/a.png", + "detail": "low", + }, + }, + ], + } + ], + }, + "image_url detail/format", + ), + "consecutive_user_messages": ( + { + "model": "grok-4-0709", + "messages": [ + {"role": "user", "content": "a"}, + {"role": "user", "content": "b"}, + ], + }, + "consecutive user messages", + ), + "empty_tools_list": ( + {"model": "grok-4-0709", "tools": [], "messages": _USER}, + "empty tools list", + ), +} + + +def _v2(case: dict): + return translate_chat_request(copy.deepcopy(case), "xai", build_real_deps()) + + +def _norm(body: dict) -> str: + return json.dumps(body, sort_keys=True, default=str) + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_v1_at_head_still_matches_the_snapshot(name: str) -> None: + snapshot = (SNAPSHOTS_DIR / "requests" / f"{name}.json").read_text() + assert canonical_json(run_v1_request_transform(CASES[name])) == snapshot + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_v2_request_matches_the_snapshot(name: str) -> None: + result = _v2(CASES[name]) + assert result.is_ok(), result.error.summary + snapshot = load_json(SNAPSHOTS_DIR / "requests" / f"{name}.json") + assert _norm(result.ok) == _norm(snapshot) + + +@pytest.mark.parametrize("name", sorted(V1_RAISES)) +def test_v1_raise_rows_fall_back_typed(name: str) -> None: + case, reason_fragment = V1_RAISES[name] + result = _v2(case) + assert result.is_error(), f"{name} unexpectedly translated: {result.ok!r}" + assert reason_fragment in result.error.summary, result.error.summary + with pytest.raises(UnsupportedParamsError): + run_v1_request_transform(case) + + +@pytest.mark.parametrize("name", sorted(EXPECTED_FALLBACKS)) +def test_unsupported_shape_is_a_typed_fallback(name: str) -> None: + case, reason_fragment = EXPECTED_FALLBACKS[name] + result = _v2(case) + assert result.is_error(), f"{name} unexpectedly translated: {result.ok!r}" + assert reason_fragment in result.error.summary, result.error.summary + + +def test_reasoning_gate_matches_v1_capability_read() -> None: + """The v2 gate reads deps.supports_capability over the ``xai/{model}`` + map key; it must agree with v1's litellm.supports_reasoning on the + models the corpus serves and falls back.""" + import litellm + + from litellm.translation.providers.xai.params import supports_reasoning + + deps = build_real_deps() + for model in ( + "grok-3", + "grok-3-mini", + "grok-4-0709", + "grok-code-fast-1", + "grok-2-1212", + ): + assert supports_reasoning(model, deps) == litellm.supports_reasoning( + model=model, custom_llm_provider="xai" + ), model diff --git a/tests/test_litellm/translation/test_differential_xai_response.py b/tests/test_litellm/translation/test_differential_xai_response.py new file mode 100644 index 0000000000..c17fa85b33 --- /dev/null +++ b/tests/test_litellm/translation/test_differential_xai_response.py @@ -0,0 +1,159 @@ +"""Differential parity for the xai response path. + +The v1 reference is ``XAIChatConfig.transform_response`` — LIVE on the +httpx path (main.py:2289), the inverse of the openai SDK route — over a +real ``httpx.Response``. Two-sided: v1 at HEAD must equal the committed +snapshot (drift guard) AND v2 (``parse_response`` -> +``serialize_response("openai")`` -> ``to_model_response("openai")``) must +equal it byte-for-byte. + +The corpus pins the five xai response behaviors: the R1 finish_reason "" +chain (v1's own ``_fix_choice_finish_reason_for_tool_calls`` is dead; +v1-as-executed emits "stop" WITH tool_calls — both sides run the same live +``map_finish_reason`` via ``Choices``), the reasoning-token fold (+ its +idempotency guard), num_sources_used -> web_search_requests (live-search +billing), total_tokens normalization, and the citations top-level +passthrough. The R4 row proves the bare wire model: NO ``xai/`` prefix ever +appears (fresh ModelResponse, the cdr ``model is None`` arm). +""" + +import copy +import json + +import pytest + +from litellm.types.utils import ModelResponse + +from litellm.translation.inbound.openai_chat import parse_request +from litellm.translation.inbound.openai_chat.response import serialize_response +from litellm.translation.providers.xai.response import parse_response +from litellm.translation_seam import build_translation_deps, to_model_response + +from ._xai_corpus import ( + SNAPSHOTS_DIR, + canonical_json, + corpus, + jsonable, + run_v1_response_transform, +) + +RESPONSES = corpus("responses") + +_REQUEST = { + "model": "grok-4-0709", + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], +} + + +def _v2_model_response(raw: dict, request_model: str) -> dict: + request = {**copy.deepcopy(_REQUEST), "model": request_model} + parsed = parse_request(request) + assert parsed.is_ok(), parsed.error.summary + response = parse_response(copy.deepcopy(raw), parsed.ok) + assert response.is_ok(), response.error.summary + body = serialize_response(response.ok, build_translation_deps(), "openai") + return to_model_response(body, ModelResponse(), usage_style="openai").model_dump() + + +def _norm(payload: object) -> str: + return json.dumps(jsonable(payload), sort_keys=True) + + +@pytest.mark.parametrize("name", sorted(RESPONSES)) +def test_v1_at_head_still_matches_the_snapshot(name: str) -> None: + row = RESPONSES[name] + snapshot = (SNAPSHOTS_DIR / "responses" / f"{name}.json").read_text() + assert canonical_json(run_v1_response_transform(row["body"], row["model"])) == ( + snapshot + ) + + +@pytest.mark.parametrize("name", sorted(RESPONSES)) +def test_v2_response_matches_the_snapshot(name: str) -> None: + row = RESPONSES[name] + snapshot = (SNAPSHOTS_DIR / "responses" / f"{name}.json").read_text() + assert canonical_json(_v2_model_response(row["body"], row["model"])) == snapshot + + +def test_finish_empty_string_yields_stop_with_tool_calls() -> None: + """R1 pinned semantically, not just byte-wise: the served finish is + "stop" AND the tool calls survive (the violated stop->tool_calls + invariant v1-as-executed exhibits).""" + row = RESPONSES["finish_empty_string_with_tool_calls"] + dumped = _v2_model_response(row["body"], row["model"]) + choice = dumped["choices"][0] + assert choice["finish_reason"] == "stop" + assert choice["message"]["tool_calls"], "tool_calls must survive the '' finish" + + +def test_no_xai_prefix_on_the_response_model() -> None: + """R4: the xai httpx path starts from a FRESH ModelResponse (model=None, + main.py:1401) and adopts the bare wire model; no seam may prefix it.""" + row = RESPONSES["text_basic"] + v1 = run_v1_response_transform(row["body"], row["model"]).model_dump() + v2 = _v2_model_response(row["body"], row["model"]) + assert v1["model"] == v2["model"] == "grok-4-0709" + assert "/" not in v2["model"] + + +def test_websearch_billing_fields_reach_usage() -> None: + """The live-search billing hook: cost_per_web_search_request reads + usage.prompt_tokens_details.web_search_requests.""" + row = RESPONSES["websearch_sources_and_citations"] + dumped = _v2_model_response(row["body"], row["model"]) + assert dumped["usage"]["prompt_tokens_details"]["web_search_requests"] == 3 + assert dumped["citations"] == ["https://a.test", "https://b.test"] + + +_UNSUPPORTED = { + "multiple_choices": ( + { + "id": "r", + "created": 1, + "model": "grok-4-0709", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"content": "a"}}, + {"index": 1, "finish_reason": "stop", "message": {"content": "b"}}, + ], + }, + "multiple response choices", + ), + "legacy_function_call_output": ( + { + "id": "r", + "created": 1, + "model": "grok-4-0709", + "choices": [ + { + "index": 0, + "finish_reason": "function_call", + "message": { + "content": None, + "role": "assistant", + "function_call": {"name": "f", "arguments": "{}"}, + }, + } + ], + }, + "function_call", + ), +} + + +@pytest.mark.parametrize("name", sorted(_UNSUPPORTED)) +def test_unreachable_response_shape_is_a_typed_error(name: str) -> None: + raw, reason_fragment = _UNSUPPORTED[name] + parsed = parse_request(copy.deepcopy(_REQUEST)) + assert parsed.is_ok() + result = parse_response(copy.deepcopy(raw), parsed.ok) + assert result.is_error(), f"{name} unexpectedly parsed" + assert reason_fragment in result.error.summary, result.error.summary diff --git a/tests/test_litellm/translation/test_differential_xai_stream.py b/tests/test_litellm/translation/test_differential_xai_stream.py new file mode 100644 index 0000000000..b76ef8f01e --- /dev/null +++ b/tests/test_litellm/translation/test_differential_xai_stream.py @@ -0,0 +1,173 @@ +"""Differential parity for xai streaming, pinned at the SSE data-line seam. + +v1 side: raw ``data:`` lines through ``XAIChatCompletionStreamingHandler`` +(the chunk_parser owns the xai BEHAVIOR: dummy-choice injection into +``choices: []`` usage tails, per-chunk usage fold/normalize, the base +handler's reasoning rename) into ``CustomStreamWrapper("xai")`` — NOT the +parsed-chunk seam the openai gate uses, because those rewrites sit below +the line seam (researcher-3 R3). v2 side: ``fold_lines`` with the xai +parser and the ``xai`` chunk dialect. Two-sided over the generated corpus: +v1 at HEAD must equal the committed snapshot AND v2 must equal it +byte-for-byte for content/reasoning/tool/finish chunks. + +The usage tail is the one pinned envelope difference (the openai-port seam +contract, inherited unchanged): v1's wrapper swallows the dummy-choice tail +mid-stream and SYNTHESIZES a final usage chunk at StopIteration (only under +``include_usage``); the v2 fold passes the wire ``choices: []`` chunk +through with the FOLDED usage, and the future streaming seam owns the +synthesis. The contract row pins byte-identical prefixes and equal usage +numbers (reasoning folded into completion on both sides). +""" + +import copy +import json + +import pytest + +from litellm.translation.engine.stream import fold_events, fold_lines +from litellm.translation.inbound.openai_chat.stream import initial_state +from litellm.translation.providers.xai.stream import parse_event, parse_line +from litellm.translation_seam import to_model_response_stream + +from ._xai_corpus import ( + SNAPSHOTS_DIR, + STREAM_MODEL, + canonical_json, + corpus, + load_json, + replay_xai_sse_lines, +) + +STREAMS = corpus("streams") +_TAIL_ROW = "usage_tail_include_usage" +_PREFIX_ROWS = sorted(name for name in STREAMS if name != _TAIL_ROW) + + +def _v2_chunks(events: list) -> list: + lines = [f"data: {json.dumps(event)}" for event in copy.deepcopy(events)] + lines.append("data: [DONE]") + folded = fold_lines(lines, parse_line, initial_state(STREAM_MODEL, dialect="xai")) + assert folded.is_ok(), folded.error.summary + return [ + to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump() + for chunk in folded.ok + ] + + +def _norm(chunks: list) -> str: + return json.dumps(chunks, sort_keys=True, default=str) + + +@pytest.mark.parametrize("name", sorted(STREAMS)) +def test_v1_at_head_still_matches_the_snapshot(name: str, frozen_ambient) -> None: + row = STREAMS[name] + snapshot = (SNAPSHOTS_DIR / "streams" / f"{name}.json").read_text() + assert ( + canonical_json(replay_xai_sse_lines(row["events"], row["stream_options"])) + == snapshot + ) + + +@pytest.mark.parametrize("name", _PREFIX_ROWS) +def test_v2_stream_matches_the_snapshot(name: str, frozen_ambient) -> None: + snapshot = load_json(SNAPSHOTS_DIR / "streams" / f"{name}.json") + assert _norm(_v2_chunks(STREAMS[name]["events"])) == _norm(snapshot) + + +def test_usage_tail_pins_the_seam_contract(frozen_ambient) -> None: + """v1's tail is the wrapper-synthesized usage chunk (envelope); v2's + tail is the wire ``choices: []`` chunk with the folded usage. Prefix + byte-identical, usage numbers equal — including the reasoning fold + (2 + 7 -> 9) and the normalized total.""" + snapshot = load_json(SNAPSHOTS_DIR / "streams" / f"{_TAIL_ROW}.json") + v2 = _v2_chunks(STREAMS[_TAIL_ROW]["events"]) + assert len(v2) == len(snapshot) + assert _norm(v2[:-1]) == _norm(snapshot[: len(v2) - 1]) + v1_tail, v2_tail = snapshot[-1], v2[-1] + assert v2_tail["choices"] == [] + for key in ("prompt_tokens", "completion_tokens", "total_tokens"): + assert v1_tail["usage"][key] == v2_tail["usage"][key], key + assert v2_tail["usage"]["completion_tokens"] == 9 # 2 + 7 folded + assert ( + v1_tail["usage"]["completion_tokens_details"]["reasoning_tokens"] + == v2_tail["usage"]["completion_tokens_details"]["reasoning_tokens"] + == 7 + ) + + +def test_usage_tail_swallowed_without_stream_options(frozen_ambient) -> None: + """Without include_usage v1 swallows the tail entirely (usage goes to + hidden params); the v2 passthrough keeps the prefix identical and the + seam owns withholding the tail — pinned so the streaming seam knows.""" + events = STREAMS[_TAIL_ROW]["events"] + v1 = replay_xai_sse_lines(events, None) + v2 = _v2_chunks(events) + assert len(v1) == len(v2) - 1 + assert _norm([c for c in map(dict, v1)]) == _norm(v2[:-1]) + + +def test_v2_line_and_event_folds_agree(frozen_ambient) -> None: + events = STREAMS["text"]["events"] + folded = fold_events( + copy.deepcopy(events), parse_event, initial_state(STREAM_MODEL, dialect="xai") + ) + assert folded.is_ok(), folded.error.summary + via_events = [ + to_model_response_stream(chunk, "chatcmpl-AMBIENT").model_dump() + for chunk in folded.ok + ] + assert _norm(via_events) == _norm(_v2_chunks(events)) + + +def _chunk(delta=None, finish=None, usage=None, choices=None): + payload = { + "id": "cmpl-u1", + "object": "chat.completion.chunk", + "created": 1718000000, + "model": STREAM_MODEL, + "choices": [ + { + "index": 0, + "delta": delta or {}, + "logprobs": None, + "finish_reason": finish, + } + ], + "usage": usage, + } + if choices is not None: + payload["choices"] = choices + return payload + + +_UNSUPPORTED_CHUNKS = { + "function_call_delta": ( + _chunk({"function_call": {"name": "f", "arguments": ""}}), + "function_call", + ), + "multiple_choices": ( + _chunk( + choices=[ + {"index": 0, "delta": {"content": "a"}, "finish_reason": None}, + {"index": 1, "delta": {"content": "b"}, "finish_reason": None}, + ] + ), + "multiple stream choices", + ), + "unknown_delta_key": ( + _chunk({"content": "x", "thinking_blocks": []}), + "stream delta keys", + ), + "error_payload": ( + {"error": {"message": "boom"}, "choices": []}, + "provider stream error", + ), +} + + +@pytest.mark.parametrize("name", sorted(_UNSUPPORTED_CHUNKS)) +def test_unreachable_chunk_shape_is_a_typed_error(name: str) -> None: + event, reason_fragment = _UNSUPPORTED_CHUNKS[name] + result = parse_event(event) + assert result.is_error(), f"{name} unexpectedly parsed" + assert reason_fragment in result.error.summary, result.error.summary