Merge pull request #24076 from Chesars/feat/cache-control-tool-config-21969

feat(bedrock): support cache_control_injection_points for tool_config location

litellm/integrations/anthropic_cache_control_hook.py

@@ -60,13 +60,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         # Create a deep copy of messages to avoid modifying the original list
         processed_messages = copy.deepcopy(messages)
 
-        # Process message-level cache controls
+        # Separate message-level and non-message-level injection points
+        remaining_points = []
         for point in injection_points:
             if point.get("location") == "message":
                 point = cast(CacheControlMessageInjectionPoint, point)
                 processed_messages = self._process_message_injection(
                     point=point, messages=processed_messages
                 )
+            else:
+                remaining_points.append(point)
+
+        # Pass through non-message injection points for provider-specific handling
+        if remaining_points:
+            non_default_params["cache_control_injection_points"] = remaining_points
 
         return model, processed_messages, non_default_params
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1446,6 +1446,16 @@ class AmazonConverseConfig(BaseConfig):
             original_tools, model, headers, additional_request_params
         )
 
+        # Append cachePoint to tools if cache_control_injection_points has tool_config
+        cache_injection_points = additional_request_params.pop(
+            "cache_control_injection_points", None
+        )
+        if cache_injection_points and len(bedrock_tools) > 0:
+            for point in cache_injection_points:
+                if point.get("location") == "tool_config":
+                    bedrock_tools.append({"cachePoint": {"type": "default"}})
+                    break
+
         bedrock_tool_config: Optional[ToolConfigBlock] = None
         if len(bedrock_tools) > 0:
             tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(

litellm/types/integrations/anthropic_cache_control_hook.py

@@ -16,4 +16,13 @@ class CacheControlMessageInjectionPoint(TypedDict):
     control: Optional[ChatCompletionCachedContent]
 
 
-CacheControlInjectionPoint = CacheControlMessageInjectionPoint
+class CacheControlToolConfigInjectionPoint(TypedDict):
+    """Type for tool_config-level injection points (Bedrock)."""
+
+    location: Literal["tool_config"]
+
+
+CacheControlInjectionPoint = Union[
+    CacheControlMessageInjectionPoint,
+    CacheControlToolConfigInjectionPoint,
+]

tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py

@@ -3803,3 +3803,100 @@ def test_streaming_without_json_mode_passes_all_tools():
     assert tool_use_delta is not None
     assert tool_use_delta["function"]["arguments"] == '{"data": 1}'
 
+
+def test_cache_control_injection_tool_config():
+    """Test that cache_control_injection_points with location=tool_config appends cachePoint to tools."""
+    config = AmazonConverseConfig()
+    messages = [
+        {"role": "user", "content": "What is the weather?"},
+    ]
+    optional_params = {
+        "tools": [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get weather for a location",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "location": {"type": "string"},
+                        },
+                        "required": ["location"],
+                    },
+                },
+            }
+        ],
+        "cache_control_injection_points": [
+            {"location": "tool_config"},
+        ],
+    }
+    result = config._transform_request(
+        model="anthropic.claude-3-5-haiku-20241022-v1:0",
+        messages=messages,
+        optional_params=optional_params,
+        litellm_params={},
+    )
+    tool_config = result["toolConfig"]
+    tools = tool_config["tools"]
+    # Last element should be a cachePoint block
+    assert tools[-1] == {"cachePoint": {"type": "default"}}
+    # First element should be the actual tool
+    assert "toolSpec" in tools[0]
+
+
+def test_cache_control_injection_tool_config_no_tools():
+    """Test that tool_config injection is ignored when no tools are provided."""
+    config = AmazonConverseConfig()
+    messages = [
+        {"role": "user", "content": "Hello"},
+    ]
+    optional_params = {
+        "cache_control_injection_points": [
+            {"location": "tool_config"},
+        ],
+    }
+    result = config._transform_request(
+        model="anthropic.claude-3-5-haiku-20241022-v1:0",
+        messages=messages,
+        optional_params=optional_params,
+        litellm_params={},
+    )
+    assert "toolConfig" not in result
+
+
+def test_cache_control_injection_tool_config_not_added_without_injection_point():
+    """Test that cachePoint is NOT appended when cache_control_injection_points doesn't include tool_config."""
+    config = AmazonConverseConfig()
+    messages = [
+        {"role": "user", "content": "What is the weather?"},
+    ]
+    optional_params = {
+        "tools": [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get weather",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"location": {"type": "string"}},
+                        "required": ["location"],
+                    },
+                },
+            }
+        ],
+        "cache_control_injection_points": [
+            {"location": "message", "role": "system"},
+        ],
+    }
+    result = config._transform_request(
+        model="anthropic.claude-3-5-haiku-20241022-v1:0",
+        messages=messages,
+        optional_params=optional_params,
+        litellm_params={},
+    )
+    tools = result["toolConfig"]["tools"]
+    # No cachePoint should be appended
+    assert all("cachePoint" not in tool for tool in tools)
+