test(translation): encode the baseten steady state and registration-coverage completeness (critic-wave1a M2, verifier-wave1a F3)

The drop canary gains a registration half: negative asserts that baseten
is absent from the dispatch Literal and all four pipeline tables, so naive
registration goes red with or without a corpus row (the verifier's variant
B experiment sailed through green before this). And a completeness gate
pins Literal == every pipeline table == dedicated-gate providers + the
compat_sdk SPECS corpus, with PROFILES == ALLOWED == SPECS, so ANY future
wave registering a provider without differential coverage fails here, not
just baseten.
This commit is contained in:
mateo-berri
2026-06-12 10:38:37 +00:00
parent c05635e9de
commit fa9e015a19
2 changed files with 50 additions and 2 deletions
@@ -13,6 +13,7 @@ the v1 configs at HEAD, over every model-map row where the provider has any.
import copy
import json
from typing import get_args
import pytest
@@ -20,6 +21,9 @@ import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.translation import translate_chat_request
from litellm.translation.dispatch import Provider
from litellm.translation.engine import pipeline
from litellm.translation.providers import compat_sdk
from litellm.translation.providers.compat_sdk import params as csp
from ._compat_sdk_corpus import (
@@ -244,6 +248,36 @@ def _request_rows():
)
def test_registered_providers_have_differential_coverage() -> None:
"""Green-but-untested registration is impossible (verifier-wave1a F3):
every dispatch Literal member is registered in every pipeline table,
the compat_sdk family registry equals the corpus SPECS exactly, and a
provider outside the family must be in the dedicated-gates set below.
Add a provider to that set ONLY in the commit that adds its
differential corpus, naming the gate files."""
providers = set(get_args(Provider))
assert providers == set(pipeline._SERIALIZERS)
assert providers == set(pipeline._RESPONSE_PARSERS)
assert providers == set(pipeline._RESPONSE_DIALECTS)
assert set(pipeline._RAW_GUARDS) <= providers
assert set(compat_sdk.PROFILES) == set(SPECS)
assert set(compat_sdk.ALLOWED) == set(SPECS)
dedicated_gates = {
"anthropic", # test_differential_anthropic_{request,response,stream}
"bedrock_converse", # test_differential_bedrock_*
"bedrock_invoke", # test_differential_bedrock_*
"openai_compat", # test_differential_openai_*
"vertex_ai", # test_differential_google_*
"gemini", # test_differential_google_*
"vertex_anthropic", # test_differential_google_*
"azure", # test_differential_azure_*
"azure_ai", # test_differential_azure_ai_request + azure stream/response
"azure_ai_anthropic", # test_differential_azure_ai_request (Claude route)
"xai", # test_differential_xai_*
}
assert providers == dedicated_gates | set(SPECS)
@pytest.mark.parametrize("provider,name", _request_rows())
def test_v2_request_matches_v1(provider: str, name: str) -> None:
case = corpus_for(provider)[name]
@@ -16,6 +16,7 @@ the bottom pins the evidence so the drop reason stays true at HEAD.
import copy
import json
import time
from typing import get_args
import pytest
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
@@ -23,6 +24,9 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.translation.dispatch import Provider
from litellm.translation.engine import pipeline
from ._compat_sdk_corpus import PROVIDERS
from .test_differential_openai_stream import MODEL, STREAMS, USAGE_STREAM, _v2_chunks
@@ -85,10 +89,20 @@ def test_baseten_drop_canary(frozen_ambient) -> None:
dedicated legacy decoder (``handle_baseten_chunk``,
streaming_handler.py:1246-1248) instead of the default openai arm, so
its v1 stream behavior is not the openai dialect and cannot be honestly
registered. If this canary fails, the legacy branch is gone at HEAD —
registered. If the DRIFT half fails, the legacy branch is gone at HEAD —
re-evaluate porting baseten (request/response sides are trivial:
own list, mct rename, user supported)."""
own list, mct rename, user supported). The REGISTRATION half makes the
steady state loud (critic-wave1a M2 / verifier F3): registering baseten
anywhere requires deliberately deleting these negative asserts, which
forces the registrar past the evidence above."""
# drift half: the legacy branch still exists and still diverges
assert hasattr(CustomStreamWrapper, "handle_baseten_chunk")
v1_baseten = _v1_chunks("baseten", STREAMS["text"])
v1_openai = _v1_chunks("custom_openai", STREAMS["text"])
assert _norm(v1_baseten) != _norm(v1_openai)
# registration half: naive registration fails HERE, corpus row or not
assert "baseten" not in get_args(Provider)
assert "baseten" not in pipeline._SERIALIZERS
assert "baseten" not in pipeline._RESPONSE_PARSERS
assert "baseten" not in pipeline._RESPONSE_DIALECTS
assert "baseten" not in pipeline._RAW_GUARDS