diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 4887cbd23b..79153c3cef 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -215,6 +215,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) + anthropic_request.pop("stream_chunk_size", None) output_format = anthropic_request.pop("output_format", None) output_config_format = pop_bedrock_invoke_output_config_format( anthropic_request diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 4385044007..6bb2da1ad4 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -150,6 +150,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): ) -> dict: ## SETUP ## stream = optional_params.pop("stream", None) + optional_params.pop("stream_chunk_size", None) custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {} hf_model_name = litellm_params.get("hf_model_name", None) diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py new file mode 100644 index 0000000000..aff89f02ff --- /dev/null +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -0,0 +1,41 @@ +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, +) +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) + + +@pytest.mark.parametrize( + "config,model", + [ + (AmazonInvokeConfig, "anthropic.claude-3-sonnet-20240229-v1:0"), + (AmazonInvokeConfig, "amazon.titan-text-express-v1"), + (AmazonInvokeConfig, "mistral.mistral-7b-instruct-v0:2"), + (AmazonAnthropicClaudeConfig, "anthropic.claude-sonnet-4-6"), + ], +) +def test_transform_request_drops_stream_chunk_size(config, model): + """stream_chunk_size is a LiteLLM-internal knob for re-chunking the HTTP + response stream. Leaking it into the provider request body makes Bedrock + reject the whole request: ValidationException 'stream_chunk_size: Extra + inputs are not permitted'.""" + request_body = config().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"stream": True, "stream_chunk_size": 2048, "max_tokens": 10}, + litellm_params={}, + headers={}, + ) + + assert "stream_chunk_size" not in json.dumps(request_body)