mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-20 04:23:56 +00:00
[Fix] Anthropic cache_control incorrectly applied to all content items instead of last item only (#15699)
* fix: _safe_insert_cache_control_in_message * test_anthropic_cache_control_hook_system_message * docs prompt cache injection * docs fix
This commit is contained in:
Ishaan Jaff
@@ -506,3 +506,11 @@ curl -L -X GET 'http://0.0.0.0:4000/v1/model/info' \
|
||||
</Tabs>
|
||||
|
||||
This checks our maintained [model info/cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
|
||||
|
||||
## Read More
|
||||
|
||||
:::tip Auto-Inject Prompt Caching
|
||||
Want LiteLLM to automatically add `cache_control` directives without modifying your code?
|
||||
|
||||
See [**Auto-Inject Prompt Caching Tutorial**](../tutorials/prompt_caching.md) to learn how to use `cache_control_injection_points` to automatically cache system messages, specific messages by index, or custom injection patterns.
|
||||
:::
|
||||
|
||||
@@ -24,15 +24,174 @@ You need to specify `cache_control_injection_points` in your model configuration
|
||||
|
||||
LiteLLM will then automatically add a `cache_control` directive to the specified messages in your requests:
|
||||
|
||||
```json
|
||||
```json showLineNumbers title="cache_control_directive.json"
|
||||
"cache_control": {
|
||||
"type": "ephemeral"
|
||||
}
|
||||
```
|
||||
|
||||
## Usage Example
|
||||
## LiteLLM Python SDK Usage
|
||||
|
||||
In this example, we'll configure caching for system messages by adding the directive to all messages with `role: system`.
|
||||
Use the `cache_control_injection_points` parameter in your completion calls to automatically inject caching directives.
|
||||
|
||||
#### Basic Example - Cache System Messages
|
||||
|
||||
```python showLineNumbers title="cache_system_messages.py"
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["ANTHROPIC_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="anthropic/claude-3-5-sonnet-20240620",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
# Auto-inject cache control to system messages
|
||||
cache_control_injection_points=[
|
||||
{
|
||||
"location": "message",
|
||||
"role": "system",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
**Key Points:**
|
||||
- Use `cache_control_injection_points` parameter to specify where to inject caching
|
||||
- `location: "message"` targets messages in the conversation
|
||||
- `role: "system"` targets all system messages
|
||||
- LiteLLM automatically adds `cache_control` to the **last content block** of matching messages (per Anthropic's API specification)
|
||||
|
||||
**LiteLLM's Modified Request:**
|
||||
|
||||
LiteLLM automatically transforms your request by adding `cache_control` to the last content block of the system message:
|
||||
|
||||
```json showLineNumbers title="modified_request_system.json"
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents."
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement...",
|
||||
"cache_control": {"type": "ephemeral"} // Added by LiteLLM
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Target Specific Messages by Index
|
||||
|
||||
You can target specific messages by their index in the messages array. Use negative indices to target from the end.
|
||||
|
||||
```python showLineNumbers title="cache_by_index.py"
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["ANTHROPIC_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="anthropic/claude-3-5-sonnet-20240620",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "First message",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Response to first",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here is a long document to analyze:"},
|
||||
{"type": "text", "text": "Document content..." * 500},
|
||||
],
|
||||
},
|
||||
],
|
||||
# Target the last message (index -1)
|
||||
cache_control_injection_points=[
|
||||
{
|
||||
"location": "message",
|
||||
"index": -1, # -1 targets the last message, -2 would target second-to-last, etc.
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
**Important Notes:**
|
||||
- When a message has multiple content blocks (like images or multiple text blocks), `cache_control` is only added to the **last content block**
|
||||
- This follows [Anthropic's API specification](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#continuing-a-multi-turn-conversation) which requires: "When using multiple content blocks, only the last content block can have cache_control"
|
||||
- Anthropic has a maximum of 4 blocks with `cache_control` per request
|
||||
|
||||
**LiteLLM's Modified Request:**
|
||||
|
||||
LiteLLM adds `cache_control` to the last content block of the targeted message (index -1 = last message):
|
||||
|
||||
```json showLineNumbers title="modified_request_index.json"
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "First message"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Response to first"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is a long document to analyze:"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Document content...",
|
||||
"cache_control": {"type": "ephemeral"} // Added by LiteLLM to last content block only
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## LiteLLM Proxy Usage
|
||||
|
||||
You can configure cache control injection in the proxy configuration file.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="litellm config.yaml" label="litellm config.yaml">
|
||||
@@ -64,7 +223,7 @@ On the LiteLLM UI, you can specify the `cache_control_injection_points` in the `
|
||||
|
||||
In this example, we have a very long, static system message and a varying user message. It's efficient to cache the system message since it rarely changes.
|
||||
|
||||
```json
|
||||
```json showLineNumbers title="original_request.json"
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
@@ -93,7 +252,7 @@ In this example, we have a very long, static system message and a varying user m
|
||||
|
||||
LiteLLM auto-injects the caching directive into the system message based on our configuration:
|
||||
|
||||
```json
|
||||
```json showLineNumbers title="modified_request.json"
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
@@ -121,8 +280,9 @@ LiteLLM auto-injects the caching directive into the system message based on our
|
||||
|
||||
When the model provider processes this request, it will recognize the caching directive and only process the system message once, caching it for subsequent requests.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
|
||||
- [Manual Prompt Caching](../completion/prompt_caching.md) - Learn how to manually add `cache_control` directives to your messages
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -120,17 +120,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
||||
- list of objects
|
||||
|
||||
This method handles inserting cache control in both cases.
|
||||
Per Anthropic's API specification, when using multiple content blocks,
|
||||
only the last content block can have cache_control.
|
||||
"""
|
||||
message_content = message.get("content", None)
|
||||
|
||||
# 1. if string, insert cache control in the message
|
||||
if isinstance(message_content, str):
|
||||
message["cache_control"] = control # type: ignore
|
||||
# 2. list of objects
|
||||
# 2. list of objects - only apply to last item per Anthropic spec
|
||||
elif isinstance(message_content, list):
|
||||
for content_item in message_content:
|
||||
if isinstance(content_item, dict):
|
||||
content_item["cache_control"] = control # type: ignore
|
||||
if len(message_content) > 0 and isinstance(message_content[-1], dict):
|
||||
message_content[-1]["cache_control"] = control # type: ignore
|
||||
return message
|
||||
|
||||
@property
|
||||
|
||||
@@ -91,8 +91,13 @@ async def test_anthropic_cache_control_hook_system_message():
|
||||
|
||||
print("request_body: ", json.dumps(request_body, indent=4))
|
||||
|
||||
# Verify the request body
|
||||
assert request_body["system"][1]["cachePoint"] == {"type": "default"}
|
||||
# Verify that cache control was applied (Bedrock transforms it to a separate item)
|
||||
cache_control_count = sum(
|
||||
1
|
||||
for item in request_body["system"]
|
||||
if isinstance(item, dict) and "cachePoint" in item
|
||||
)
|
||||
assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -753,3 +758,148 @@ async def test_anthropic_cache_control_hook_no_op():
|
||||
for item in content
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_cache_control_hook_multiple_content_items_last_only():
|
||||
"""
|
||||
Test that cache_control is only applied to the last content item in a list, not all items.
|
||||
This verifies the fix for https://github.com/BerriAI/litellm/issues/15696
|
||||
"""
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"AWS_ACCESS_KEY_ID": "fake_access_key_id",
|
||||
"AWS_SECRET_ACCESS_KEY": "fake_secret_access_key",
|
||||
"AWS_REGION_NAME": "us-west-2",
|
||||
},
|
||||
):
|
||||
anthropic_cache_control_hook = AnthropicCacheControlHook()
|
||||
litellm.callbacks = [anthropic_cache_control_hook]
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Response",
|
||||
}
|
||||
},
|
||||
"stopReason": "stop_sequence",
|
||||
"usage": {
|
||||
"inputTokens": 100,
|
||||
"outputTokens": 200,
|
||||
"totalTokens": 300,
|
||||
},
|
||||
}
|
||||
mock_response.status_code = 200
|
||||
|
||||
client = AsyncHTTPHandler()
|
||||
with patch.object(client, "post", return_value=mock_response) as mock_post:
|
||||
response = await litellm.acompletion(
|
||||
model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "First piece of context"},
|
||||
{"type": "text", "text": "Second piece of context"},
|
||||
{"type": "text", "text": "Third piece of context"},
|
||||
{"type": "text", "text": "Fourth piece of context"},
|
||||
{"type": "text", "text": "Fifth piece of context - should be cached"},
|
||||
],
|
||||
}
|
||||
],
|
||||
cache_control_injection_points=[
|
||||
{"location": "message", "index": -1}
|
||||
],
|
||||
client=client,
|
||||
)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
request_body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
|
||||
print("Multi-content request_body: ", json.dumps(request_body, indent=4))
|
||||
|
||||
message_content = request_body["messages"][0]["content"]
|
||||
assert isinstance(message_content, list)
|
||||
|
||||
cache_control_count = sum(
|
||||
1
|
||||
for item in message_content
|
||||
if isinstance(item, dict) and "cachePoint" in item
|
||||
)
|
||||
assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_cache_control_hook_document_analysis_multiple_pages():
|
||||
"""
|
||||
Test cache_control with multiple document pages to ensure only the last page gets cached.
|
||||
This simulates document analysis with 6 content blocks, verifying the fix for issue 15696.
|
||||
"""
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"AWS_ACCESS_KEY_ID": "fake_access_key_id",
|
||||
"AWS_SECRET_ACCESS_KEY": "fake_secret_access_key",
|
||||
"AWS_REGION_NAME": "us-west-2",
|
||||
},
|
||||
):
|
||||
anthropic_cache_control_hook = AnthropicCacheControlHook()
|
||||
litellm.callbacks = [anthropic_cache_control_hook]
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Summary",
|
||||
}
|
||||
},
|
||||
"stopReason": "stop_sequence",
|
||||
"usage": {
|
||||
"inputTokens": 100,
|
||||
"outputTokens": 200,
|
||||
"totalTokens": 300,
|
||||
},
|
||||
}
|
||||
mock_response.status_code = 200
|
||||
|
||||
client = AsyncHTTPHandler()
|
||||
with patch.object(client, "post", return_value=mock_response) as mock_post:
|
||||
response = await litellm.acompletion(
|
||||
model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this document"},
|
||||
{"type": "text", "text": "Page 1 content"},
|
||||
{"type": "text", "text": "Page 2 content"},
|
||||
{"type": "text", "text": "Page 3 content"},
|
||||
{"type": "text", "text": "Page 4 content"},
|
||||
{"type": "text", "text": "Page 5 content - final page to cache"},
|
||||
],
|
||||
}
|
||||
],
|
||||
cache_control_injection_points=[
|
||||
{"location": "message", "role": "user"}
|
||||
],
|
||||
client=client,
|
||||
)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
request_body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
|
||||
print("Document analysis request_body: ", json.dumps(request_body, indent=4))
|
||||
|
||||
message_content = request_body["messages"][0]["content"]
|
||||
assert isinstance(message_content, list)
|
||||
|
||||
cache_control_count = sum(
|
||||
1
|
||||
for item in message_content
|
||||
if isinstance(item, dict) and "cachePoint" in item
|
||||
)
|
||||
assert cache_control_count == 1, f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)."
|
||||
|
||||
Reference in New Issue
Block a user