diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index a0d63f5043..11fdb26e42 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -75,7 +75,7 @@ "effort-2025-11-24": "effort-2025-11-24", "fast-mode-2026-02-01": null, "files-api-2025-04-14": null, - "fine-grained-tool-streaming-2025-05-14": null, + "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14", "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, @@ -106,7 +106,7 @@ "effort-2025-11-24": "effort-2025-11-24", "fast-mode-2026-02-01": null, "files-api-2025-04-14": null, - "fine-grained-tool-streaming-2025-05-14": null, + "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14", "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 388947a4e9..7e1020000f 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -32,7 +32,7 @@ def make_sync_call( logging_obj: LiteLLMLoggingObject, json_mode: Optional[bool] = False, fake_stream: bool = False, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ): if client is None: client = _get_httpx_client() # Create a new client if none provided @@ -108,7 +108,7 @@ class BedrockConverseLLM(BaseAWSLLM): fake_stream: bool = False, json_mode: Optional[bool] = False, api_key: Optional[str] = None, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ) -> CustomStreamWrapper: request_data = await litellm.AmazonConverseConfig()._async_transform_request( model=model, @@ -268,7 +268,7 @@ class BedrockConverseLLM(BaseAWSLLM): ): ## SETUP ## stream = optional_params.pop("stream", None) - stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) + stream_chunk_size = optional_params.pop("stream_chunk_size", None) unencoded_model_id = optional_params.pop("model_id", None) fake_stream = optional_params.pop("fake_stream", False) json_mode = optional_params.get("json_mode", False) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 7a9916f1f3..0a1322a751 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -197,7 +197,7 @@ async def make_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ): try: if client is None: @@ -294,7 +294,7 @@ def make_sync_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ): try: if client is None: @@ -790,7 +790,7 @@ class BedrockLLM(BaseAWSLLM): ## SETUP ## stream = optional_params.pop("stream", None) - stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) + stream_chunk_size = optional_params.pop("stream_chunk_size", None) provider = self.get_bedrock_invoke_provider(model) modelId = self.get_bedrock_model_id( @@ -1203,7 +1203,7 @@ class BedrockLLM(BaseAWSLLM): extra_headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ) -> Union[ModelResponse, CustomStreamWrapper]: transformed_request = ( await litellm.AmazonAnthropicClaudeConfig().async_transform_request( @@ -1350,7 +1350,7 @@ class BedrockLLM(BaseAWSLLM): logger_fn=None, headers={}, client: Optional[AsyncHTTPHandler] = None, - stream_chunk_size: int = 1024, + stream_chunk_size: Optional[int] = None, ) -> CustomStreamWrapper: # The call is not made here; instead, we prepare the necessary objects for the stream. diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index a415d55021..61987d25d9 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -1,12 +1,21 @@ import os import sys +from unittest.mock import AsyncMock, MagicMock +import pytest sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path -from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder +import litellm +from litellm.llms.bedrock.chat.invoke_handler import ( + AWSEventStreamDecoder, + BedrockLLM, + make_call, + make_sync_call, +) +from litellm.llms.custom_httpx.http_handler import HTTPHandler def test_transform_thinking_blocks_with_redacted_content(): @@ -200,3 +209,120 @@ def test_bedrock_converse_streaming_consistent_id(): assert ( response.id == expected_id ), "All chunk IDs must match the one captured from the messageStart event" + + +@pytest.mark.asyncio +async def test_make_call_does_not_rechunk_stream_by_default(): + """Re-chunking the event stream into fixed 1024-byte blocks holds small + early events (messageStart, contentBlockStart) in httpx's ByteChunker until + 1024 bytes accumulate, delaying time-to-first-chunk by the whole generation + when Bedrock trickles bytes (e.g. buffered tool-use streams).""" + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = AsyncMock(return_value=response) + + await make_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + ) + + response.aiter_bytes.assert_called_once_with(chunk_size=None) + + +@pytest.mark.asyncio +async def test_make_call_honors_explicit_stream_chunk_size(): + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = AsyncMock(return_value=response) + + await make_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + stream_chunk_size=2048, + ) + + response.aiter_bytes.assert_called_once_with(chunk_size=2048) + + +def test_make_sync_call_does_not_rechunk_stream_by_default(): + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = MagicMock(return_value=response) + + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + signed_json_body=None, + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + ) + + response.iter_bytes.assert_called_once_with(chunk_size=None) + + +def test_make_sync_call_honors_explicit_stream_chunk_size(): + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = MagicMock(return_value=response) + + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + signed_json_body=None, + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + stream_chunk_size=2048, + ) + + response.iter_bytes.assert_called_once_with(chunk_size=2048) + + +def test_legacy_bedrock_llm_streaming_does_not_rechunk_by_default(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_bytes = MagicMock(return_value=iter([])) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + BedrockLLM().completion( + model="cohere.command-text-v14", + messages=[{"role": "user", "content": "hi"}], + api_base=None, + custom_prompt_dict={}, + model_response=litellm.ModelResponse(), + print_verbose=lambda *args, **kwargs: None, + encoding=litellm.encoding, + logging_obj=MagicMock(), + optional_params={ + "stream": True, + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "aws_region_name": "us-east-1", + }, + acompletion=False, + timeout=None, + litellm_params={}, + client=client, + ) + + mock_response.iter_bytes.assert_called_once_with(chunk_size=None) diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index b636ea468c..2a3db5982e 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,10 +1,14 @@ import os import sys +from unittest.mock import MagicMock import pytest +import litellm from litellm.llms.bedrock.chat import BedrockConverseLLM +from litellm.llms.bedrock.chat.converse_handler import make_sync_call from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions +from litellm.llms.custom_httpx.http_handler import HTTPHandler sys.path.insert( 0, os.path.abspath("../../../../..") @@ -133,3 +137,79 @@ class TestBedrockRegionInModelPath: assert model_id == "moonshotai.kimi-k2.5" # explicitly set region is preserved assert optional_params["aws_region_name"] == "eu-west-1" + + +def _stream_completion_with_spied_iter_bytes(model: str, **kwargs) -> MagicMock: + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_bytes = MagicMock(return_value=iter([])) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + **kwargs, + ) + return mock_response.iter_bytes + + +def test_make_sync_call_does_not_rechunk_stream_by_default(): + """Re-chunking the event stream into fixed 1024-byte blocks holds small + early events in httpx's ByteChunker until 1024 bytes accumulate, delaying + time-to-first-chunk by the whole generation when Bedrock trickles bytes + (e.g. buffered tool-use streams).""" + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = MagicMock(return_value=response) + + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + ) + + response.iter_bytes.assert_called_once_with(chunk_size=None) + + +def test_make_sync_call_honors_explicit_stream_chunk_size(): + response = MagicMock() + response.status_code = 200 + client = MagicMock() + client.post = MagicMock(return_value=response) + + make_sync_call( + client=client, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream", + headers={}, + data="{}", + model="anthropic.claude-sonnet-4-6", + messages=[], + logging_obj=MagicMock(), + stream_chunk_size=2048, + ) + + response.iter_bytes.assert_called_once_with(chunk_size=2048) + + +def test_completion_plumbs_stream_chunk_size_through_converse(): + iter_bytes_spy = _stream_completion_with_spied_iter_bytes( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" + ) + iter_bytes_spy.assert_called_once_with(chunk_size=None) + + iter_bytes_spy = _stream_completion_with_spied_iter_bytes( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + stream_chunk_size=2048, + ) + iter_bytes_spy.assert_called_once_with(chunk_size=2048) diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 9cd27e88c3..59bab22de7 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -412,6 +412,20 @@ class TestAnthropicBetaHeadersFiltering: assert filtered == ["compact-2026-01-12"] + @pytest.mark.parametrize("provider", ["bedrock_converse", "bedrock"]) + def test_fine_grained_tool_streaming_forwarded_for_bedrock(self, provider): + """Bedrock honors fine-grained-tool-streaming-2025-05-14 via + additionalModelRequestFields.anthropic_beta. Stripping it (previously + mapped to null) silently re-enables Anthropic's server-side buffering of + tool-call argument deltas, so streamed tool args arrive in a single + end-of-stream burst instead of incrementally.""" + filtered = filter_and_transform_beta_headers( + beta_headers=["fine-grained-tool-streaming-2025-05-14"], + provider=provider, + ) + + assert filtered == ["fine-grained-tool-streaming-2025-05-14"] + def test_null_value_headers_filtered(self): """Test that headers with null values are always filtered out.""" for provider in [