fix(bedrock): stop buffering streamed tool-call argument deltas (#30231)

* fix(bedrock): stop buffering streamed tool-call argument deltas

Two issues made Bedrock tool-use streaming arrive as a single end-of-stream
burst through LiteLLM while plain text streamed fine.

First, the anthropic-beta allowlist mapped fine-grained-tool-streaming-2025-05-14
to null for bedrock and bedrock_converse, so the header was silently stripped.
Without that beta, Anthropic models on Bedrock buffer tool input server-side and
emit all toolUse.input deltas at once (verified against converse-stream and
invoke-with-response-stream directly). Bedrock accepts the beta via
additionalModelRequestFields.anthropic_beta, so it is now forwarded.

Second, the streaming reads re-chunked the AWS event stream with
iter_bytes(chunk_size=1024). httpx's ByteChunker only releases full 1024-byte
blocks, so the small early events (messageStart, contentBlockStart, first
deltas) sat in the buffer until enough bytes accumulated, pushing
time-to-first-byte from ~1.4s to ~8.5s on buffered tool-use streams. The
default is now no re-chunking; an explicit stream_chunk_size is still honored.

* test(bedrock): cover explicit stream_chunk_size on sync invoke path

* test(bedrock): cover stream_chunk_size plumbing through converse completion

* test(bedrock): cover stream_chunk_size default in legacy BedrockLLM streaming

* test(bedrock): merge converse handler tests into existing mapped test file

pytest imports test modules by basename in non-package test dirs, so the new
tests/test_litellm/llms/bedrock/chat/test_converse_handler.py collided with
the pre-existing tests/test_litellm/llms/chat/test_converse_handler.py and
broke collection in CI. Move the new tests into the existing file

litellm/anthropic_beta_headers_config.json

@@ -75,7 +75,7 @@
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": null,
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,
@@ -106,7 +106,7 @@
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": null,
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,

litellm/llms/bedrock/chat/converse_handler.py

@@ -32,7 +32,7 @@ def make_sync_call(
     logging_obj: LiteLLMLoggingObject,
     json_mode: Optional[bool] = False,
     fake_stream: bool = False,
-    stream_chunk_size: int = 1024,
+    stream_chunk_size: Optional[int] = None,
 ):
     if client is None:
         client = _get_httpx_client()  # Create a new client if none provided
@@ -108,7 +108,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         fake_stream: bool = False,
         json_mode: Optional[bool] = False,
         api_key: Optional[str] = None,
-        stream_chunk_size: int = 1024,
+        stream_chunk_size: Optional[int] = None,
     ) -> CustomStreamWrapper:
         request_data = await litellm.AmazonConverseConfig()._async_transform_request(
             model=model,
@@ -268,7 +268,7 @@ class BedrockConverseLLM(BaseAWSLLM):
     ):
         ## SETUP ##
         stream = optional_params.pop("stream", None)
-        stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
+        stream_chunk_size = optional_params.pop("stream_chunk_size", None)
         unencoded_model_id = optional_params.pop("model_id", None)
         fake_stream = optional_params.pop("fake_stream", False)
         json_mode = optional_params.get("json_mode", False)

litellm/llms/bedrock/chat/invoke_handler.py

@@ -197,7 +197,7 @@ async def make_call(
     fake_stream: bool = False,
     json_mode: Optional[bool] = False,
     bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
-    stream_chunk_size: int = 1024,
+    stream_chunk_size: Optional[int] = None,
 ):
     try:
         if client is None:
@@ -294,7 +294,7 @@ def make_sync_call(
     fake_stream: bool = False,
     json_mode: Optional[bool] = False,
     bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
-    stream_chunk_size: int = 1024,
+    stream_chunk_size: Optional[int] = None,
 ):
     try:
         if client is None:
@@ -790,7 +790,7 @@ class BedrockLLM(BaseAWSLLM):
 
         ## SETUP ##
         stream = optional_params.pop("stream", None)
-        stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
+        stream_chunk_size = optional_params.pop("stream_chunk_size", None)
 
         provider = self.get_bedrock_invoke_provider(model)
         modelId = self.get_bedrock_model_id(
@@ -1203,7 +1203,7 @@ class BedrockLLM(BaseAWSLLM):
         extra_headers: Optional[dict] = None,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
         client: Optional[AsyncHTTPHandler] = None,
-        stream_chunk_size: int = 1024,
+        stream_chunk_size: Optional[int] = None,
     ) -> Union[ModelResponse, CustomStreamWrapper]:
         transformed_request = (
             await litellm.AmazonAnthropicClaudeConfig().async_transform_request(
@@ -1350,7 +1350,7 @@ class BedrockLLM(BaseAWSLLM):
         logger_fn=None,
         headers={},
         client: Optional[AsyncHTTPHandler] = None,
-        stream_chunk_size: int = 1024,
+        stream_chunk_size: Optional[int] = None,
     ) -> CustomStreamWrapper:
         # The call is not made here; instead, we prepare the necessary objects for the stream.
 

tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py

@@ -1,12 +1,21 @@
 import os
 import sys
+from unittest.mock import AsyncMock, MagicMock
 
+import pytest
 
 sys.path.insert(
     0, os.path.abspath("../../../../..")
 )  # Adds the parent directory to the system path
 
-from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
+import litellm
+from litellm.llms.bedrock.chat.invoke_handler import (
+    AWSEventStreamDecoder,
+    BedrockLLM,
+    make_call,
+    make_sync_call,
+)
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
 def test_transform_thinking_blocks_with_redacted_content():
@@ -200,3 +209,120 @@ def test_bedrock_converse_streaming_consistent_id():
         assert (
             response.id == expected_id
         ), "All chunk IDs must match the one captured from the messageStart event"
+
+
+@pytest.mark.asyncio
+async def test_make_call_does_not_rechunk_stream_by_default():
+    """Re-chunking the event stream into fixed 1024-byte blocks holds small
+    early events (messageStart, contentBlockStart) in httpx's ByteChunker until
+    1024 bytes accumulate, delaying time-to-first-chunk by the whole generation
+    when Bedrock trickles bytes (e.g. buffered tool-use streams)."""
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = AsyncMock(return_value=response)
+
+    await make_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+    )
+
+    response.aiter_bytes.assert_called_once_with(chunk_size=None)
+
+
+@pytest.mark.asyncio
+async def test_make_call_honors_explicit_stream_chunk_size():
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = AsyncMock(return_value=response)
+
+    await make_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+        stream_chunk_size=2048,
+    )
+
+    response.aiter_bytes.assert_called_once_with(chunk_size=2048)
+
+
+def test_make_sync_call_does_not_rechunk_stream_by_default():
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = MagicMock(return_value=response)
+
+    make_sync_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        signed_json_body=None,
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+    )
+
+    response.iter_bytes.assert_called_once_with(chunk_size=None)
+
+
+def test_make_sync_call_honors_explicit_stream_chunk_size():
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = MagicMock(return_value=response)
+
+    make_sync_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        signed_json_body=None,
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+        stream_chunk_size=2048,
+    )
+
+    response.iter_bytes.assert_called_once_with(chunk_size=2048)
+
+
+def test_legacy_bedrock_llm_streaming_does_not_rechunk_by_default():
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.iter_bytes = MagicMock(return_value=iter([]))
+    client = HTTPHandler()
+    client.post = MagicMock(return_value=mock_response)
+
+    BedrockLLM().completion(
+        model="cohere.command-text-v14",
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=None,
+        custom_prompt_dict={},
+        model_response=litellm.ModelResponse(),
+        print_verbose=lambda *args, **kwargs: None,
+        encoding=litellm.encoding,
+        logging_obj=MagicMock(),
+        optional_params={
+            "stream": True,
+            "aws_access_key_id": "fake",
+            "aws_secret_access_key": "fake",
+            "aws_region_name": "us-east-1",
+        },
+        acompletion=False,
+        timeout=None,
+        litellm_params={},
+        client=client,
+    )
+
+    mock_response.iter_bytes.assert_called_once_with(chunk_size=None)

tests/test_litellm/llms/chat/test_converse_handler.py

@@ -1,10 +1,14 @@
 import os
 import sys
+from unittest.mock import MagicMock
 
 import pytest
 
+import litellm
 from litellm.llms.bedrock.chat import BedrockConverseLLM
+from litellm.llms.bedrock.chat.converse_handler import make_sync_call
 from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 sys.path.insert(
     0, os.path.abspath("../../../../..")
@@ -133,3 +137,79 @@ class TestBedrockRegionInModelPath:
         assert model_id == "moonshotai.kimi-k2.5"
         # explicitly set region is preserved
         assert optional_params["aws_region_name"] == "eu-west-1"
+
+
+def _stream_completion_with_spied_iter_bytes(model: str, **kwargs) -> MagicMock:
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.iter_bytes = MagicMock(return_value=iter([]))
+    client = HTTPHandler()
+    client.post = MagicMock(return_value=mock_response)
+
+    litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        client=client,
+        aws_access_key_id="fake",
+        aws_secret_access_key="fake",
+        aws_region_name="us-east-1",
+        **kwargs,
+    )
+    return mock_response.iter_bytes
+
+
+def test_make_sync_call_does_not_rechunk_stream_by_default():
+    """Re-chunking the event stream into fixed 1024-byte blocks holds small
+    early events in httpx's ByteChunker until 1024 bytes accumulate, delaying
+    time-to-first-chunk by the whole generation when Bedrock trickles bytes
+    (e.g. buffered tool-use streams)."""
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = MagicMock(return_value=response)
+
+    make_sync_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+    )
+
+    response.iter_bytes.assert_called_once_with(chunk_size=None)
+
+
+def test_make_sync_call_honors_explicit_stream_chunk_size():
+    response = MagicMock()
+    response.status_code = 200
+    client = MagicMock()
+    client.post = MagicMock(return_value=response)
+
+    make_sync_call(
+        client=client,
+        api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-sonnet-4-6/converse-stream",
+        headers={},
+        data="{}",
+        model="anthropic.claude-sonnet-4-6",
+        messages=[],
+        logging_obj=MagicMock(),
+        stream_chunk_size=2048,
+    )
+
+    response.iter_bytes.assert_called_once_with(chunk_size=2048)
+
+
+def test_completion_plumbs_stream_chunk_size_through_converse():
+    iter_bytes_spy = _stream_completion_with_spied_iter_bytes(
+        model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
+    )
+    iter_bytes_spy.assert_called_once_with(chunk_size=None)
+
+    iter_bytes_spy = _stream_completion_with_spied_iter_bytes(
+        model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
+        stream_chunk_size=2048,
+    )
+    iter_bytes_spy.assert_called_once_with(chunk_size=2048)

tests/test_litellm/test_anthropic_beta_headers_filtering.py

@@ -412,6 +412,20 @@ class TestAnthropicBetaHeadersFiltering:
 
         assert filtered == ["compact-2026-01-12"]
 
+    @pytest.mark.parametrize("provider", ["bedrock_converse", "bedrock"])
+    def test_fine_grained_tool_streaming_forwarded_for_bedrock(self, provider):
+        """Bedrock honors fine-grained-tool-streaming-2025-05-14 via
+        additionalModelRequestFields.anthropic_beta. Stripping it (previously
+        mapped to null) silently re-enables Anthropic's server-side buffering of
+        tool-call argument deltas, so streamed tool args arrive in a single
+        end-of-stream burst instead of incrementally."""
+        filtered = filter_and_transform_beta_headers(
+            beta_headers=["fine-grained-tool-streaming-2025-05-14"],
+            provider=provider,
+        )
+
+        assert filtered == ["fine-grained-tool-streaming-2025-05-14"]
+
     def test_null_value_headers_filtered(self):
         """Test that headers with null values are always filtered out."""
         for provider in [