From dad0894dff13e18dca0e907ff5584983c672f51f Mon Sep 17 00:00:00 2001 From: Kent Date: Thu, 11 Jun 2026 03:03:22 +0000 Subject: [PATCH] feat(bedrock_mantle): route Responses API to native OpenAI endpoint (#29490) Backport prerequisite for #29788. Applied as the squash diff of PR #29490 (head 50ab150fa6^..), which landed upstream inside the litellm_oss_staging_040626 sync (cb041966bf, #29671) and has no standalone commit to cherry-pick. --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 5 + .../llms/bedrock_mantle/responses/__init__.py | 0 .../responses/transformation.py | 81 +++++ ...odel_prices_and_context_window_backup.json | 38 +++ litellm/utils.py | 10 + model_prices_and_context_window.json | 38 +++ ...bedrock_mantle_responses_transformation.py | 283 ++++++++++++++++++ 8 files changed, 458 insertions(+) create mode 100644 litellm/llms/bedrock_mantle/responses/__init__.py create mode 100644 litellm/llms/bedrock_mantle/responses/transformation.py create mode 100644 tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 7c92623358..e56e9f5435 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1722,6 +1722,9 @@ if TYPE_CHECKING: from .llms.openrouter.responses.transformation import ( OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig, ) + from .llms.bedrock_mantle.responses.transformation import ( + BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, + ) from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 17eb660929..9fd4302f36 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -237,6 +237,7 @@ LLM_CONFIG_NAMES = ( "PerplexityResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", + "BedrockMantleResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "OpenAIOSeriesConfig", "AnthropicSkillsConfig", @@ -956,6 +957,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.openrouter.responses.transformation", "OpenRouterResponsesAPIConfig", ), + "BedrockMantleResponsesAPIConfig": ( + ".llms.bedrock_mantle.responses.transformation", + "BedrockMantleResponsesAPIConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", diff --git a/litellm/llms/bedrock_mantle/responses/__init__.py b/litellm/llms/bedrock_mantle/responses/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py new file mode 100644 index 0000000000..b63fd0ecdb --- /dev/null +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -0,0 +1,81 @@ +""" +Amazon Bedrock Mantle - Responses API backend. + +gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses` +path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI +Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides +only the endpoint URL and Bearer authentication. + +Auth: AWS Bedrock API key as Bearer token (BEDROCK_MANTLE_API_KEY or the +standard AWS_BEARER_TOKEN_BEDROCK), NOT SigV4. +""" + +from typing import Optional + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1" + +# Checked longest/most-specific first so a full endpoint URL collapses to host +# in one pass and the appended path never doubles. +_BASE_SUFFIXES_TO_STRIP = ( + "/openai/v1/responses", + "/v1/responses", + "/responses", + "/openai/v1", + "/v1", +) + + +class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK_MANTLE + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + region = ( + get_secret_str("BEDROCK_MANTLE_REGION") + or get_secret_str("AWS_REGION") + or BEDROCK_MANTLE_DEFAULT_REGION + ) + base = ( + api_base + or get_secret_str("BEDROCK_MANTLE_API_BASE") + or f"https://bedrock-mantle.{region}.api.aws" + ) + base = base.rstrip("/") + for suffix in _BASE_SUFFIXES_TO_STRIP: + if base.endswith(suffix): + base = base[: -len(suffix)] + break + return f"{base}/openai/v1/responses" + + def validate_environment( + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or get_secret_str("BEDROCK_MANTLE_API_KEY") + or get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + ) + if not api_key: + raise ValueError( + "Bedrock Mantle API key is required. Set BEDROCK_MANTLE_API_KEY " + "(or AWS_BEARER_TOKEN_BEDROCK) or pass api_key." + ) + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cdd8f8c215..cf81488090 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -40988,6 +40988,44 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "bedrock_mantle/openai.gpt-5.5": { + "input_cost_per_token": 5.5e-06, + "cache_read_input_token_cost": 5.5e-07, + "output_cost_per_token": 3.3e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/openai.gpt-5.4": { + "input_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 2.75e-07, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, diff --git a/litellm/utils.py b/litellm/utils.py index 2ba6ef9cae..79d34e42e9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8807,6 +8807,16 @@ class ProviderConfigManager: return litellm.OpenRouterResponsesAPIConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMResponsesAPIConfig() + elif litellm.LlmProviders.BEDROCK_MANTLE == provider: + # Only OpenAI gpt frontier models (gpt-5.x, and future gpt-6 etc.) are + # served on the /openai/v1/responses path. gpt-oss and every non-OpenAI + # model on Mantle (nvidia, mistral, google, zai, ...) are chat-completions + # only and 400 on that path, so they fall through to None to keep the + # chat-completions emulation (see litellm/responses/main.py "config is None"). + model_lower = model.lower() if model else "" + if "openai.gpt-" in model_lower and "gpt-oss" not in model_lower: + return litellm.BedrockMantleResponsesAPIConfig() + return None return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7aa3195195..5238822b1a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41142,6 +41142,44 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "bedrock_mantle/openai.gpt-5.5": { + "input_cost_per_token": 5.5e-06, + "cache_read_input_token_cost": 5.5e-07, + "output_cost_per_token": 3.3e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/openai.gpt-5.4": { + "input_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 2.75e-07, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py new file mode 100644 index 0000000000..e2133d56f8 --- /dev/null +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -0,0 +1,283 @@ +""" +Unit tests for Amazon Bedrock Mantle Responses API configuration. + +Mantle's gpt-5.5 / gpt-5.4 are served ONLY on the non-standard +`/openai/v1/responses` path. These tests lock the URL construction and +Bearer auth that make that routing work. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import pytest + +import litellm +from litellm.llms.bedrock_mantle.responses.transformation import ( + BedrockMantleResponsesAPIConfig, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class TestBedrockMantleResponsesURL: + def test_url_uses_region_from_env(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + + def test_url_normalizes_v1_suffix(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url( + api_base="https://bedrock-mantle.us-east-2.api.aws/v1", + litellm_params={}, + ) + assert url == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + assert "/v1/openai/v1/responses" not in url + url_trailing = cfg.get_complete_url( + api_base="https://bedrock-mantle.us-east-2.api.aws/v1/", + litellm_params={}, + ) + assert ( + url_trailing + == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + ) + + def test_url_does_not_double_openai_v1(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url( + api_base="https://bedrock-mantle.us-east-2.api.aws/openai/v1", + litellm_params={}, + ) + assert url == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + + def test_url_full_endpoint_base_not_doubled(self, monkeypatch): + # AWS model card tells users to set OPENAI_BASE_URL to the full endpoint. + # If copied into api_base, it must not be doubled. + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url( + api_base="https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses", + litellm_params={}, + ) + assert url == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + assert url.count("/responses") == 1 + + def test_url_region_fallback_to_aws_region(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + monkeypatch.setenv("AWS_REGION", "us-west-2") + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://bedrock-mantle.us-west-2.api.aws/openai/v1/responses" + + def test_url_region_default_us_east_1(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses" + + +class TestBedrockMantleResponsesAuth: + def test_config_api_key_takes_priority(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") + cfg = BedrockMantleResponsesAPIConfig() + headers = cfg.validate_environment( + headers={}, + model="openai.gpt-5.5", + litellm_params=GenericLiteLLMParams(api_key="config-key"), + ) + assert headers["Authorization"] == "Bearer config-key" + + def test_env_key_fallback(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer env-key" + + def test_bedrock_bearer_token_fallback(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "bearer-key") + cfg = BedrockMantleResponsesAPIConfig() + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer bearer-key" + + def test_missing_key_raises(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + with pytest.raises(ValueError, match="Bedrock Mantle API key"): + cfg.validate_environment( + headers={}, + model="openai.gpt-5.5", + litellm_params=GenericLiteLLMParams(), + ) + + def test_custom_llm_provider(self): + cfg = BedrockMantleResponsesAPIConfig() + assert cfg.custom_llm_provider == LlmProviders.BEDROCK_MANTLE + + def test_native_websocket_disabled(self): + # Mantle Responses has no realtime/websocket transport, so the config + # must opt out; otherwise realtime routing would try a socket Mantle + # does not serve. + cfg = BedrockMantleResponsesAPIConfig() + assert cfg.supports_native_websocket() is False + + def test_file_search_routes_to_emulation(self): + # Mantle cannot reach OpenAI's vector stores, so a native file_search + # tool forwarded as-is gets a 400. The config must opt out of native + # file_search so LiteLLM's emulation handles it instead of forwarding. + from litellm.responses.file_search.emulated_handler import ( + should_use_emulated_file_search, + ) + + cfg = BedrockMantleResponsesAPIConfig() + assert cfg.supports_native_file_search() is False + assert ( + should_use_emulated_file_search( + tools=[{"type": "file_search", "vector_store_ids": ["vs_1"]}], + provider_config=cfg, + ) + is True + ) + + +class TestBedrockMantleResponsesRegistry: + def test_registry_returns_config_for_gpt_5_5(self): + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model="openai.gpt-5.5", + ) + assert isinstance(cfg, BedrockMantleResponsesAPIConfig) + + def test_registry_returns_config_for_gpt_5_4_enum(self): + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider=LlmProviders.BEDROCK_MANTLE, + model="openai.gpt-5.4", + ) + assert isinstance(cfg, BedrockMantleResponsesAPIConfig) + + def test_registry_returns_none_for_gpt_oss(self): + # Regression guard: gpt-oss must NOT get the native Responses config; it + # keeps the chat-completions emulation path (responses/main.py ~line 1109). + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model="openai.gpt-oss-120b", + ) + assert cfg is None + + def test_registry_returns_none_for_gpt_oss_safeguard(self): + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model="openai.gpt-oss-safeguard-20b", + ) + assert cfg is None + + def test_registry_returns_config_for_future_frontier_model(self): + # Forward-compatibility: an unseen OpenAI gpt frontier model (e.g. gpt-6) must + # get the native Responses config without a code change. The gate allow-lists + # the openai.gpt- family (minus gpt-oss), so gpt-6 matches automatically. + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model="openai.gpt-6", + ) + assert isinstance(cfg, BedrockMantleResponsesAPIConfig) + + @pytest.mark.parametrize( + "model", + [ + "nvidia.nemotron-nano-9b-v2", + "mistral.ministral-3-3b-instruct", + "google.gemma-3-27b-it", + "zai.glm-4.6", + ], + ) + def test_registry_returns_none_for_non_openai_models(self, model): + # Regression for the chat-only families on Mantle. These models 400 on + # /openai/v1/responses and are served on /v1/chat/completions, so the + # registry must NOT hand them the Responses config; they fall through to + # None and keep the chat-completions emulation. + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model=model, + ) + assert cfg is None + + def test_registry_returns_none_when_model_is_none(self): + # By-id operations (delete/get/cancel) call with model=None; keep returning + # None so those paths are unchanged. + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_responses_api_config( + provider="bedrock_mantle", + model=None, + ) + assert cfg is None + + +@pytest.fixture +def local_cost_map(monkeypatch): + """Force the bundled backup cost map and re-derive the provider model sets. + + ``litellm.model_cost`` is populated once at import time (here, from the + network-fetched ``main`` copy, which lags this branch). ``add_known_models`` + only re-buckets whatever is already in ``model_cost``, so the cost map must + first be reloaded from the local backup before the new keys appear. + """ + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + litellm.add_known_models() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +class TestBedrockMantleResponsesPricing: + def test_gpt_5_5_pricing_and_mode(self, local_cost_map): + info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.5") + assert info["mode"] == "responses" + assert info["input_cost_per_token"] == pytest.approx(5.5e-06) + assert info["output_cost_per_token"] == pytest.approx(3.3e-05) + assert info["cache_read_input_token_cost"] == pytest.approx(5.5e-07) + assert info["max_input_tokens"] == 272000 + + def test_gpt_5_4_pricing_and_mode(self, local_cost_map): + info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.4") + assert info["mode"] == "responses" + assert info["input_cost_per_token"] == pytest.approx(2.75e-06) + assert info["output_cost_per_token"] == pytest.approx(1.65e-05) + assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07) + assert info["max_input_tokens"] == 272000 + + def test_models_registered(self, local_cost_map): + assert "bedrock_mantle/openai.gpt-5.5" in litellm.bedrock_mantle_models + assert "bedrock_mantle/openai.gpt-5.4" in litellm.bedrock_mantle_models