mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-05 06:22:12 +00:00
test(translation): encode the baseten steady state and registration-coverage completeness (critic-wave1a M2, verifier-wave1a F3)
The drop canary gains a registration half: negative asserts that baseten is absent from the dispatch Literal and all four pipeline tables, so naive registration goes red with or without a corpus row (the verifier's variant B experiment sailed through green before this). And a completeness gate pins Literal == every pipeline table == dedicated-gate providers + the compat_sdk SPECS corpus, with PROFILES == ALLOWED == SPECS, so ANY future wave registering a provider without differential coverage fails here, not just baseten.
This commit is contained in:
@@ -13,6 +13,7 @@ the v1 configs at HEAD, over every model-map row where the provider has any.
|
||||
|
||||
import copy
|
||||
import json
|
||||
from typing import get_args
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -20,6 +21,9 @@ import litellm
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
|
||||
from litellm.translation import translate_chat_request
|
||||
from litellm.translation.dispatch import Provider
|
||||
from litellm.translation.engine import pipeline
|
||||
from litellm.translation.providers import compat_sdk
|
||||
from litellm.translation.providers.compat_sdk import params as csp
|
||||
|
||||
from ._compat_sdk_corpus import (
|
||||
@@ -244,6 +248,36 @@ def _request_rows():
|
||||
)
|
||||
|
||||
|
||||
def test_registered_providers_have_differential_coverage() -> None:
|
||||
"""Green-but-untested registration is impossible (verifier-wave1a F3):
|
||||
every dispatch Literal member is registered in every pipeline table,
|
||||
the compat_sdk family registry equals the corpus SPECS exactly, and a
|
||||
provider outside the family must be in the dedicated-gates set below.
|
||||
Add a provider to that set ONLY in the commit that adds its
|
||||
differential corpus, naming the gate files."""
|
||||
providers = set(get_args(Provider))
|
||||
assert providers == set(pipeline._SERIALIZERS)
|
||||
assert providers == set(pipeline._RESPONSE_PARSERS)
|
||||
assert providers == set(pipeline._RESPONSE_DIALECTS)
|
||||
assert set(pipeline._RAW_GUARDS) <= providers
|
||||
assert set(compat_sdk.PROFILES) == set(SPECS)
|
||||
assert set(compat_sdk.ALLOWED) == set(SPECS)
|
||||
dedicated_gates = {
|
||||
"anthropic", # test_differential_anthropic_{request,response,stream}
|
||||
"bedrock_converse", # test_differential_bedrock_*
|
||||
"bedrock_invoke", # test_differential_bedrock_*
|
||||
"openai_compat", # test_differential_openai_*
|
||||
"vertex_ai", # test_differential_google_*
|
||||
"gemini", # test_differential_google_*
|
||||
"vertex_anthropic", # test_differential_google_*
|
||||
"azure", # test_differential_azure_*
|
||||
"azure_ai", # test_differential_azure_ai_request + azure stream/response
|
||||
"azure_ai_anthropic", # test_differential_azure_ai_request (Claude route)
|
||||
"xai", # test_differential_xai_*
|
||||
}
|
||||
assert providers == dedicated_gates | set(SPECS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider,name", _request_rows())
|
||||
def test_v2_request_matches_v1(provider: str, name: str) -> None:
|
||||
case = corpus_for(provider)[name]
|
||||
|
||||
@@ -16,6 +16,7 @@ the bottom pins the evidence so the drop reason stays true at HEAD.
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
from typing import get_args
|
||||
|
||||
import pytest
|
||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||
@@ -23,6 +24,9 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
from litellm.translation.dispatch import Provider
|
||||
from litellm.translation.engine import pipeline
|
||||
|
||||
from ._compat_sdk_corpus import PROVIDERS
|
||||
from .test_differential_openai_stream import MODEL, STREAMS, USAGE_STREAM, _v2_chunks
|
||||
|
||||
@@ -85,10 +89,20 @@ def test_baseten_drop_canary(frozen_ambient) -> None:
|
||||
dedicated legacy decoder (``handle_baseten_chunk``,
|
||||
streaming_handler.py:1246-1248) instead of the default openai arm, so
|
||||
its v1 stream behavior is not the openai dialect and cannot be honestly
|
||||
registered. If this canary fails, the legacy branch is gone at HEAD —
|
||||
registered. If the DRIFT half fails, the legacy branch is gone at HEAD —
|
||||
re-evaluate porting baseten (request/response sides are trivial:
|
||||
own list, mct rename, user supported)."""
|
||||
own list, mct rename, user supported). The REGISTRATION half makes the
|
||||
steady state loud (critic-wave1a M2 / verifier F3): registering baseten
|
||||
anywhere requires deliberately deleting these negative asserts, which
|
||||
forces the registrar past the evidence above."""
|
||||
# drift half: the legacy branch still exists and still diverges
|
||||
assert hasattr(CustomStreamWrapper, "handle_baseten_chunk")
|
||||
v1_baseten = _v1_chunks("baseten", STREAMS["text"])
|
||||
v1_openai = _v1_chunks("custom_openai", STREAMS["text"])
|
||||
assert _norm(v1_baseten) != _norm(v1_openai)
|
||||
# registration half: naive registration fails HERE, corpus row or not
|
||||
assert "baseten" not in get_args(Provider)
|
||||
assert "baseten" not in pipeline._SERIALIZERS
|
||||
assert "baseten" not in pipeline._RESPONSE_PARSERS
|
||||
assert "baseten" not in pipeline._RESPONSE_DIALECTS
|
||||
assert "baseten" not in pipeline._RAW_GUARDS
|
||||
|
||||
Reference in New Issue
Block a user