From 433d1a494780af26f875cfb2f9994eaf903a5c4e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 25 Aug 2025 13:44:54 -0700 Subject: [PATCH] [Bug fix] - Fix /messages fallback from Anthropic API -> Bedrock API (#13946) * use helper get_provider_specific_headers * fix get_provider_specific_headers * test_anthropic_messages_fallbacks * bedrock/us.anthropic.claude-sonnet-4 * fix: get_provider_specific_headers * TestProviderSpecificHeaderUtils * test_anthropic_messages_fallbacks --- .../get_provider_specific_headers.py | 23 ++++++++ litellm/llms/custom_httpx/llm_http_handler.py | 11 ++-- litellm/main.py | 13 +++-- litellm/proxy/proxy_config.yaml | 24 +++++---- .../test_anthropic_messages_passthrough.py | 52 +++++++++++++++++++ .../test_provider_specific_headers.py | 43 +++++++++++++++ 6 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 litellm/litellm_core_utils/get_provider_specific_headers.py create mode 100644 tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py new file mode 100644 index 0000000000..cf9165cfda --- /dev/null +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -0,0 +1,23 @@ +from typing import Dict, Optional + +from litellm.types.utils import ProviderSpecificHeader + + +class ProviderSpecificHeaderUtils: + @staticmethod + def get_provider_specific_headers( + provider_specific_header: Optional[ProviderSpecificHeader], + custom_llm_provider: Optional[str], + ) -> Dict: + """ + Get the provider specific headers for the given custom llm provider + + Returns: + Optional[Dict]: The provider specific headers for the given custom llm provider + """ + if ( + provider_specific_header is not None + and provider_specific_header.get("custom_llm_provider") == custom_llm_provider + ): + return provider_specific_header.get("extra_headers", {}) + return {} \ No newline at end of file diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c9d70088d0..d404077a5b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1257,6 +1257,10 @@ class BaseLLMHTTPHandler: stream: Optional[bool] = False, kwargs: Optional[Dict[str, Any]] = None, ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, + ) + if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders.ANTHROPIC @@ -1270,10 +1274,9 @@ class BaseLLMHTTPHandler: Optional[litellm.types.utils.ProviderSpecificHeader], kwargs.get("provider_specific_header", None), ) - extra_headers = ( - provider_specific_header.get("extra_headers", {}) - if provider_specific_header - else {} + extra_headers = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, + custom_llm_provider=custom_llm_provider, ) ( headers, diff --git a/litellm/main.py b/litellm/main.py index c8442d483e..6102fe3ccc 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -61,6 +61,9 @@ from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_for_health_check from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, +) from litellm.litellm_core_utils.health_check_utils import ( _create_health_check_response, _filter_model_params, @@ -1107,11 +1110,11 @@ def completion( # type: ignore # noqa: PLR0915 api_key=api_key, ) - if ( - provider_specific_header is not None - and provider_specific_header["custom_llm_provider"] == custom_llm_provider - ): - headers.update(provider_specific_header["extra_headers"]) + if provider_specific_header is not None: + headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, + custom_llm_provider=custom_llm_provider, + )) if model_response is not None and hasattr(model_response, "_hidden_params"): model_response._hidden_params["custom_llm_provider"] = custom_llm_provider diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 28ba8cd093..5748502a50 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,17 +1,23 @@ model_list: - - model_name: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + - model_name: anthropic/* litellm_params: - model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + model: anthropic/* + api_key: os.environ/OPENAI_API_KEY_IJ - model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 litellm_params: model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 + - model_name: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + +router_settings: + fallbacks: [ + {"anthropic/claude-opus-4-20250514": + { + "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + } + } + ] litellm_settings: callbacks: ["datadog_llm_observability"] -guardrails: - - guardrail_name: "bedrock-pre-guard" - litellm_params: - guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" - mode: "during_call" - guardrailIdentifier: ff6ujrregl1q - guardrailVersion: "DRAFT" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 098daf7893..ee73efb4a3 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -273,6 +273,58 @@ async def test_anthropic_messages_litellm_router_routing_strategy(): print(f"Non-streaming response: {json.dumps(response, indent=2)}") return response +@pytest.mark.asyncio +async def test_anthropic_messages_fallbacks(): + """ + E2E test the anthropic_messages fallbacks from Anthropic API to Bedrock + """ + litellm._turn_on_debug() + router = Router( + model_list=[ + { + "model_name": "anthropic/claude-opus-4-20250514", + "litellm_params": { + "model": "anthropic/claude-opus-4-20250514", + "api_key": "bad-key", + }, + }, + { + "model_name": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + }, + } + ], + fallbacks=[ + { + "anthropic/claude-opus-4-20250514": + ["bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"] + } + ] + ) + + # Set up test parameters + messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] + + # Call the handler + response = await router.aanthropic_messages( + messages=messages, + model="anthropic/claude-opus-4-20250514", + max_tokens=100, + metadata={ + "user_id": "hello", + }, + ) + + # Verify response + assert "id" in response + assert "content" in response + assert "model" in response + assert response["role"] == "assistant" + + print(f"Non-streaming response: {json.dumps(response, indent=2)}") + return response + @pytest.mark.asyncio async def test_anthropic_messages_litellm_router_latency_metadata_tracking(): diff --git a/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py b/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py new file mode 100644 index 0000000000..aa1d31c616 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py @@ -0,0 +1,43 @@ +import pytest + +from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, +) +from litellm.types.utils import ProviderSpecificHeader + + +class TestProviderSpecificHeaderUtils: + def test_get_provider_specific_headers_matching_provider(self): + """Test that the method returns extra_headers when custom_llm_provider matches.""" + provider_specific_header: ProviderSpecificHeader = { + "custom_llm_provider": "openai", + "extra_headers": {"Authorization": "Bearer token123", "Custom-Header": "value"} + } + custom_llm_provider = "openai" + + result = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header, custom_llm_provider + ) + + expected = {"Authorization": "Bearer token123", "Custom-Header": "value"} + assert result == expected + + def test_get_provider_specific_headers_no_match_or_none(self): + """Test that the method returns empty dict when provider doesn't match or is None.""" + # Test case 1: Provider doesn't match + provider_specific_header: ProviderSpecificHeader = { + "custom_llm_provider": "anthropic", + "extra_headers": {"Authorization": "Bearer token123"} + } + custom_llm_provider = "openai" + + result = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header, custom_llm_provider + ) + assert result == {} + + # Test case 2: provider_specific_header is None + result = ProviderSpecificHeaderUtils.get_provider_specific_headers( + None, "openai" + ) + assert result == {}