From b78495d3982672db0a4e4db71785c445a03d2385 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 14 Aug 2025 16:50:05 -0700 Subject: [PATCH] [Fix] Ensure /messages works when using `bedrock/converse/ with LiteLLM (#13627) * get_bedrock_provider_config_for_messages_api * fixes for get_bedrock_provider_config_for_messages_api * test_anthropic_messages_litellm_router_bedrock * fix merge conflicts * fix - refactor based on jugal's comment --- litellm/llms/bedrock/common_utils.py | 82 ++++++++++++++++--- litellm/proxy/proxy_config.yaml | 6 ++ litellm/utils.py | 3 +- .../test_bedrock_anthropic_messages_test.py | 68 +++++++++++++++ 4 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index e122517698..c76fc0a80c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -4,11 +4,14 @@ Common utilities used across bedrock chat/embedding/image generation import json import os -from typing import TYPE_CHECKING, List, Literal, Optional, Union +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union import httpx import litellm +from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, +) from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret @@ -443,23 +446,82 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Get the bedrock route for the given model. """ + route_mappings: Dict[str, Literal["invoke", "converse_like", "converse", "agent"]] = { + "invoke/": "invoke", + "converse_like/": "converse_like", + "converse/": "converse", + "agent/": "agent" + } + + # Check explicit routes first + for prefix, route_type in route_mappings.items(): + if prefix in model: + return route_type + base_model = BedrockModelInfo.get_base_model(model) alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model) - if "invoke/" in model: - return "invoke" - elif "converse_like" in model: - return "converse_like" - elif "converse/" in model: - return "converse" - elif "agent/" in model: - return "agent" - elif ( + if ( base_model in litellm.bedrock_converse_models or alt_model in litellm.bedrock_converse_models ): return "converse" return "invoke" + + @staticmethod + def _explicit_converse_route(model: str) -> bool: + """ + Check if the model is an explicit converse route. + """ + return "converse/" in model + + @staticmethod + def _explicit_invoke_route(model: str) -> bool: + """ + Check if the model is an explicit invoke route. + """ + return "invoke/" in model + + @staticmethod + def _explicit_agent_route(model: str) -> bool: + """ + Check if the model is an explicit agent route. + """ + return "agent/" in model + + @staticmethod + def _explicit_converse_like_route(model: str) -> bool: + """ + Check if the model is an explicit converse like route. + """ + return "converse_like/" in model + + @staticmethod + def get_bedrock_provider_config_for_messages_api(model: str) -> Optional[BaseAnthropicMessagesConfig]: + """ + Get the bedrock provider config for the given model. + + Only route to AmazonAnthropicClaude3MessagesConfig() for BaseMessagesConfig + + All other routes should return None since they will go through litellm.completion + """ + + ######################################################### + # Converse routes should go through litellm.completion() + if BedrockModelInfo._explicit_converse_route(model): + return None + + ######################################################### + # This goes through litellm.AmazonAnthropicClaude3MessagesConfig() + # Since bedrock Invoke supports Native Anthropic Messages API + ######################################################### + if "claude" in model: + return litellm.AmazonAnthropicClaudeMessagesConfig() + + ######################################################### + # These routes will go through litellm.completion() + ######################################################### + return None class BedrockEventStreamDecoderBase: """ diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 755fe82118..73643fffe8 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,3 +2,9 @@ model_list: - model_name: vertex_ai/* litellm_params: model: vertex_ai/* + - model_name: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + - model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 + litellm_params: + model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 diff --git a/litellm/utils.py b/litellm/utils.py index 708e6ab81f..2ae038e6e7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7071,7 +7071,8 @@ class ProviderConfigManager: # The 'BEDROCK' provider corresponds to Amazon's implementation of Anthropic Claude v3. # This mapping ensures that the correct configuration is returned for BEDROCK. elif litellm.LlmProviders.BEDROCK == provider: - return litellm.AmazonAnthropicClaudeMessagesConfig() + from litellm.llms.bedrock.common_utils import BedrockModelInfo + return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model) elif litellm.LlmProviders.VERTEX_AI == provider: if "claude" in model: from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( diff --git a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py new file mode 100644 index 0000000000..41edc8572c --- /dev/null +++ b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py @@ -0,0 +1,68 @@ + +import json +import os +import sys +from datetime import datetime +from typing import AsyncIterator, Dict, Any +import asyncio +import unittest.mock +from unittest.mock import AsyncMock, MagicMock +import pytest +from litellm.router import Router + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path +import litellm +from base_anthropic_unified_messages_test import BaseAnthropicMessagesTest + +INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST = BaseAnthropicMessagesTest() + +@pytest.mark.asyncio +async def test_anthropic_messages_litellm_router_bedrock(): + """ + Test the anthropic_messages with non-streaming request + """ + + litellm._turn_on_debug() + router = Router( + model_list=[ + { + "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_params": { + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + }, + }, + { + "model_name": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + }, + } + ] + ) + + # Set up test parameters + messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] + + # Call 1 using bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + response = await router.aanthropic_messages( + messages=messages, + model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + max_tokens=100, + ) + + # Verify response + INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) + + # Call 2 using bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 + response = await router.aanthropic_messages( + messages=messages, + model="bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + max_tokens=100, + ) + + # Verify response + INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) + +