diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5a95d12f5b..1dfa6d11fb 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5097,12 +5097,25 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str: return valid_string -def add_cache_point_tool_block(tool: dict) -> Optional[BedrockToolBlock]: +def add_cache_point_tool_block( + tool: dict, model: Optional[str] = None +) -> Optional[BedrockToolBlock]: + from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock + cache_control = tool.get("cache_control", None) if cache_control is not None: cache_point = cache_control.get("type", "ephemeral") if cache_point == "ephemeral": - return {"cachePoint": {"type": "default"}} + cache_point_block: CachePointBlock = {"type": "default"} + if isinstance(cache_control, dict) and "ttl" in cache_control: + ttl = cache_control["ttl"] + if ( + ttl in ["5m", "1h"] + and model is not None + and is_claude_4_5_on_bedrock(model) + ): + cache_point_block["ttl"] = ttl + return {"cachePoint": cache_point_block} return None @@ -5132,7 +5145,9 @@ def _is_bedrock_tool_block(tool: dict) -> bool: ) -def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]: +def _bedrock_tools_pt( + tools: List, model: Optional[str] = None +) -> List[BedrockToolBlock]: """ OpenAI tools looks like: tools = [ @@ -5248,7 +5263,7 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]: tool_block_list.append(tool_block) ## ADD CACHE POINT TOOL BLOCK ## - cache_point_tool_block = add_cache_point_tool_block(tool) + cache_point_tool_block = add_cache_point_tool_block(tool, model=model) if cache_point_tool_block is not None: tool_block_list.append(cache_point_tool_block) @@ -5315,9 +5330,7 @@ def default_response_schema_prompt(response_schema: dict) -> str: prompt_str = """Use this JSON schema: ```json {} - ```""".format( - response_schema - ) + ```""".format(response_schema) return prompt_str diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index db6784d042..a27153365d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1299,7 +1299,7 @@ class AmazonConverseConfig(BaseConfig): ) # Process regular function tools using existing logic - bedrock_tools = _bedrock_tools_pt(regular_tools) + bedrock_tools = _bedrock_tools_pt(regular_tools, model=model) # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) if computer_use_tools: @@ -1367,7 +1367,7 @@ class AmazonConverseConfig(BaseConfig): additional_request_params["tools"] = transformed_computer_tools else: # No computer use tools, process all tools as regular tools - bedrock_tools = _bedrock_tools_pt(filtered_tools) + bedrock_tools = _bedrock_tools_pt(filtered_tools, model=model) # Append pre-formatted tools (systemTool etc.) after transformation bedrock_tools.extend(pre_formatted_tools) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 96593b35d0..1b15ebaa76 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -132,7 +132,7 @@ class AmazonAnthropicClaudeMessagesConfig( - `scope` (e.g., "global") - always removed - `ttl` - removed for older models; Claude 4.5+ supports "5m" and "1h" - Processes both `system` and `messages` content blocks. + Processes `tools`, `system`, and `messages` content blocks. Args: anthropic_messages_request: The request dictionary to modify in-place @@ -159,6 +159,12 @@ class AmazonAnthropicClaudeMessagesConfig( if isinstance(item, dict) and "cache_control" in item: _sanitize_cache_control(item["cache_control"]) + # Process tools + if "tools" in anthropic_messages_request: + for tool in anthropic_messages_request["tools"]: + if isinstance(tool, dict) and "cache_control" in tool: + _sanitize_cache_control(tool["cache_control"]) + # Process system (list of content blocks) if "system" in anthropic_messages_request: system = anthropic_messages_request["system"] diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8fdbd3bde3..72cfd89408 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2367,3 +2367,112 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): assert text_block["type"] == "text" assert "cache_control" in text_block assert text_block["cache_control"]["type"] == "ephemeral" + + +def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(): + """ + Tools with cache_control ttl should preserve the ttl in the cachePoint + block for Claude 4.5+ models on Bedrock, matching the behavior of system + block cache_control. + + Without this fix, tool cachePoint is always {"type": "default"} (5m), + while system blocks can have ttl="1h", violating Bedrock's non-increasing + TTL ordering constraint (tools -> system -> messages). + + Ref: https://github.com/BerriAI/litellm/issues/XXXXX + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + add_cache_point_tool_block, + ) + + tool_with_1h = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + + # Claude 4.5 model: ttl should be preserved + result = add_cache_point_tool_block( + tool_with_1h, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + ) + assert result is not None + assert result["cachePoint"]["type"] == "default" + assert result["cachePoint"]["ttl"] == "1h" + + # Claude 4.5 model with 5m ttl: also preserved + tool_with_5m = { + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + result_5m = add_cache_point_tool_block( + tool_with_5m, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + ) + assert result_5m is not None + assert result_5m["cachePoint"]["ttl"] == "5m" + + # Older model: ttl should be stripped + result_old = add_cache_point_tool_block( + tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + assert result_old is not None + assert result_old["cachePoint"]["type"] == "default" + assert "ttl" not in result_old["cachePoint"] + + # No model provided: ttl should be stripped (safe default) + result_no_model = add_cache_point_tool_block(tool_with_1h, model=None) + assert result_no_model is not None + assert "ttl" not in result_no_model["cachePoint"] + + # No cache_control: returns None (unchanged behavior) + tool_no_cache = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + } + assert add_cache_point_tool_block(tool_no_cache) is None + + # cache_control without ttl: returns default cachePoint (unchanged behavior) + tool_no_ttl = {"cache_control": {"type": "ephemeral"}} + result_no_ttl = add_cache_point_tool_block( + tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + ) + assert result_no_ttl is not None + assert result_no_ttl["cachePoint"]["type"] == "default" + assert "ttl" not in result_no_ttl["cachePoint"] + + +def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): + """ + End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl + for Claude 4.5+ models when tools have cache_control with ttl. + """ + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] + + # Claude 4.5: cachePoint should have ttl + result = _bedrock_tools_pt( + tools, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + ) + cache_blocks = [b for b in result if "cachePoint" in b] + assert len(cache_blocks) == 1 + assert cache_blocks[0]["cachePoint"]["ttl"] == "1h" + + # Older model: cachePoint should not have ttl + result_old = _bedrock_tools_pt( + tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + cache_blocks_old = [b for b in result_old if "cachePoint" in b] + assert len(cache_blocks_old) == 1 + assert "ttl" not in cache_blocks_old[0]["cachePoint"] diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 7a2a6f56d6..93d56d4cd0 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -467,6 +467,86 @@ def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_o assert result["tools"][0]["type"] == "custom" +def test_remove_ttl_from_cache_control_processes_tools(): + """ + Ensure _remove_ttl_from_cache_control also sanitizes cache_control on tools. + + Without this, tools keep unsupported ttl values while system/messages have + them stripped, causing TTL ordering violations on Bedrock. + """ + + cfg = AmazonAnthropicClaudeMessagesConfig() + + # Tools with ttl should have it stripped for non-Claude-4.5 models + request = { + "tools": [ + { + "name": "get_weather", + "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }, + { + "name": "get_time", + "input_schema": {"type": "object"}, + }, + ], + "system": [ + { + "type": "text", + "text": "You are helpful.", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + "messages": [], + } + + cfg._remove_ttl_from_cache_control( + request, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + + # Tool ttl should be stripped + assert "ttl" not in request["tools"][0]["cache_control"] + assert request["tools"][0]["cache_control"]["type"] == "ephemeral" + # Tool without cache_control should be unchanged + assert "cache_control" not in request["tools"][1] + # System ttl should also be stripped + assert "ttl" not in request["system"][0]["cache_control"] + + +def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(): + """ + For Claude 4.5+ models, ttl in ["5m", "1h"] should be preserved on tools, + just like it is for system and messages. + """ + + cfg = AmazonAnthropicClaudeMessagesConfig() + + request = { + "tools": [ + { + "name": "get_weather", + "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }, + ], + "system": [ + { + "type": "text", + "text": "You are helpful.", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + + cfg._remove_ttl_from_cache_control( + request, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + ) + + # Both tools and system should preserve ttl for Claude 4.5 + assert request["tools"][0]["cache_control"]["ttl"] == "1h" + assert request["system"][0]["cache_control"]["ttl"] == "1h" + + def test_remove_scope_from_cache_control(): """Ensure scope field is removed from cache_control for Bedrock (not supported)."""