diff --git a/docs/my-website/docs/completion/prompt_caching.md b/docs/my-website/docs/completion/prompt_caching.md index 9447a11d52..c8adf4bccc 100644 --- a/docs/my-website/docs/completion/prompt_caching.md +++ b/docs/my-website/docs/completion/prompt_caching.md @@ -506,3 +506,11 @@ curl -L -X GET 'http://0.0.0.0:4000/v1/model/info' \ This checks our maintained [model info/cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) + +## Read More + +:::tip Auto-Inject Prompt Caching +Want LiteLLM to automatically add `cache_control` directives without modifying your code? + +See [**Auto-Inject Prompt Caching Tutorial**](../tutorials/prompt_caching.md) to learn how to use `cache_control_injection_points` to automatically cache system messages, specific messages by index, or custom injection patterns. +::: diff --git a/docs/my-website/docs/tutorials/prompt_caching.md b/docs/my-website/docs/tutorials/prompt_caching.md index bf3d5a8dda..ab2aa00d77 100644 --- a/docs/my-website/docs/tutorials/prompt_caching.md +++ b/docs/my-website/docs/tutorials/prompt_caching.md @@ -24,15 +24,174 @@ You need to specify `cache_control_injection_points` in your model configuration LiteLLM will then automatically add a `cache_control` directive to the specified messages in your requests: -```json +```json showLineNumbers title="cache_control_directive.json" "cache_control": { "type": "ephemeral" } ``` -## Usage Example +## LiteLLM Python SDK Usage -In this example, we'll configure caching for system messages by adding the directive to all messages with `role: system`. +Use the `cache_control_injection_points` parameter in your completion calls to automatically inject caching directives. + +#### Basic Example - Cache System Messages + +```python showLineNumbers title="cache_system_messages.py" +from litellm import completion +import os + +os.environ["ANTHROPIC_API_KEY"] = "" + +response = completion( + model="anthropic/claude-3-5-sonnet-20240620", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + }, + ], + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?", + }, + ], + # Auto-inject cache control to system messages + cache_control_injection_points=[ + { + "location": "message", + "role": "system", + } + ], +) + +print(response.usage) +``` + +**Key Points:** +- Use `cache_control_injection_points` parameter to specify where to inject caching +- `location: "message"` targets messages in the conversation +- `role: "system"` targets all system messages +- LiteLLM automatically adds `cache_control` to the **last content block** of matching messages (per Anthropic's API specification) + +**LiteLLM's Modified Request:** + +LiteLLM automatically transforms your request by adding `cache_control` to the last content block of the system message: + +```json showLineNumbers title="modified_request_system.json" +{ + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents." + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement...", + "cache_control": {"type": "ephemeral"} // Added by LiteLLM + } + ] + }, + { + "role": "user", + "content": "what are the key terms and conditions in this agreement?" + } + ] +} +``` + +#### Target Specific Messages by Index + +You can target specific messages by their index in the messages array. Use negative indices to target from the end. + +```python showLineNumbers title="cache_by_index.py" +from litellm import completion +import os + +os.environ["ANTHROPIC_API_KEY"] = "" + +response = completion( + model="anthropic/claude-3-5-sonnet-20240620", + messages=[ + { + "role": "user", + "content": "First message", + }, + { + "role": "assistant", + "content": "Response to first", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Here is a long document to analyze:"}, + {"type": "text", "text": "Document content..." * 500}, + ], + }, + ], + # Target the last message (index -1) + cache_control_injection_points=[ + { + "location": "message", + "index": -1, # -1 targets the last message, -2 would target second-to-last, etc. + } + ], +) + +print(response.usage) +``` + +**Important Notes:** +- When a message has multiple content blocks (like images or multiple text blocks), `cache_control` is only added to the **last content block** +- This follows [Anthropic's API specification](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#continuing-a-multi-turn-conversation) which requires: "When using multiple content blocks, only the last content block can have cache_control" +- Anthropic has a maximum of 4 blocks with `cache_control` per request + +**LiteLLM's Modified Request:** + +LiteLLM adds `cache_control` to the last content block of the targeted message (index -1 = last message): + +```json showLineNumbers title="modified_request_index.json" +{ + "messages": [ + { + "role": "user", + "content": "First message" + }, + { + "role": "assistant", + "content": "Response to first" + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Here is a long document to analyze:" + }, + { + "type": "text", + "text": "Document content...", + "cache_control": {"type": "ephemeral"} // Added by LiteLLM to last content block only + } + ] + } + ] +} +``` + +## LiteLLM Proxy Usage + +You can configure cache control injection in the proxy configuration file. @@ -64,7 +223,7 @@ On the LiteLLM UI, you can specify the `cache_control_injection_points` in the ` In this example, we have a very long, static system message and a varying user message. It's efficient to cache the system message since it rarely changes. -```json +```json showLineNumbers title="original_request.json" { "messages": [ { @@ -93,7 +252,7 @@ In this example, we have a very long, static system message and a varying user m LiteLLM auto-injects the caching directive into the system message based on our configuration: -```json +```json showLineNumbers title="modified_request.json" { "messages": [ { @@ -121,8 +280,9 @@ LiteLLM auto-injects the caching directive into the system message based on our When the model provider processes this request, it will recognize the caching directive and only process the system message once, caching it for subsequent requests. +## Related Documentation - +- [Manual Prompt Caching](../completion/prompt_caching.md) - Learn how to manually add `cache_control` directives to your messages diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index c1fb45b304..89a93ad273 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -120,17 +120,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): - list of objects This method handles inserting cache control in both cases. + Per Anthropic's API specification, when using multiple content blocks, + only the last content block can have cache_control. """ message_content = message.get("content", None) # 1. if string, insert cache control in the message if isinstance(message_content, str): message["cache_control"] = control # type: ignore - # 2. list of objects + # 2. list of objects - only apply to last item per Anthropic spec elif isinstance(message_content, list): - for content_item in message_content: - if isinstance(content_item, dict): - content_item["cache_control"] = control # type: ignore + if len(message_content) > 0 and isinstance(message_content[-1], dict): + message_content[-1]["cache_control"] = control # type: ignore return message @property diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index dcaaf52760..38baaedef1 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -91,8 +91,13 @@ async def test_anthropic_cache_control_hook_system_message(): print("request_body: ", json.dumps(request_body, indent=4)) - # Verify the request body - assert request_body["system"][1]["cachePoint"] == {"type": "default"} + # Verify that cache control was applied (Bedrock transforms it to a separate item) + cache_control_count = sum( + 1 + for item in request_body["system"] + if isinstance(item, dict) and "cachePoint" in item + ) + assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}" @pytest.mark.asyncio @@ -753,3 +758,148 @@ async def test_anthropic_cache_control_hook_no_op(): for item in content if isinstance(item, dict) ) + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_multiple_content_items_last_only(): + """ + Test that cache_control is only applied to the last content item in a list, not all items. + This verifies the fix for https://github.com/BerriAI/litellm/issues/15696 + """ + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": "fake_access_key_id", + "AWS_SECRET_ACCESS_KEY": "fake_secret_access_key", + "AWS_REGION_NAME": "us-west-2", + }, + ): + anthropic_cache_control_hook = AnthropicCacheControlHook() + litellm.callbacks = [anthropic_cache_control_hook] + + mock_response = MagicMock() + mock_response.json.return_value = { + "output": { + "message": { + "role": "assistant", + "content": "Response", + } + }, + "stopReason": "stop_sequence", + "usage": { + "inputTokens": 100, + "outputTokens": 200, + "totalTokens": 300, + }, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + response = await litellm.acompletion( + model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "First piece of context"}, + {"type": "text", "text": "Second piece of context"}, + {"type": "text", "text": "Third piece of context"}, + {"type": "text", "text": "Fourth piece of context"}, + {"type": "text", "text": "Fifth piece of context - should be cached"}, + ], + } + ], + cache_control_injection_points=[ + {"location": "message", "index": -1} + ], + client=client, + ) + + mock_post.assert_called_once() + request_body = json.loads(mock_post.call_args.kwargs["data"]) + + print("Multi-content request_body: ", json.dumps(request_body, indent=4)) + + message_content = request_body["messages"][0]["content"] + assert isinstance(message_content, list) + + cache_control_count = sum( + 1 + for item in message_content + if isinstance(item, dict) and "cachePoint" in item + ) + assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items." + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(): + """ + Test cache_control with multiple document pages to ensure only the last page gets cached. + This simulates document analysis with 6 content blocks, verifying the fix for issue 15696. + """ + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": "fake_access_key_id", + "AWS_SECRET_ACCESS_KEY": "fake_secret_access_key", + "AWS_REGION_NAME": "us-west-2", + }, + ): + anthropic_cache_control_hook = AnthropicCacheControlHook() + litellm.callbacks = [anthropic_cache_control_hook] + + mock_response = MagicMock() + mock_response.json.return_value = { + "output": { + "message": { + "role": "assistant", + "content": "Summary", + } + }, + "stopReason": "stop_sequence", + "usage": { + "inputTokens": 100, + "outputTokens": 200, + "totalTokens": 300, + }, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + response = await litellm.acompletion( + model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document"}, + {"type": "text", "text": "Page 1 content"}, + {"type": "text", "text": "Page 2 content"}, + {"type": "text", "text": "Page 3 content"}, + {"type": "text", "text": "Page 4 content"}, + {"type": "text", "text": "Page 5 content - final page to cache"}, + ], + } + ], + cache_control_injection_points=[ + {"location": "message", "role": "user"} + ], + client=client, + ) + + mock_post.assert_called_once() + request_body = json.loads(mock_post.call_args.kwargs["data"]) + + print("Document analysis request_body: ", json.dumps(request_body, indent=4)) + + message_content = request_body["messages"][0]["content"] + assert isinstance(message_content, list) + + cache_control_count = sum( + 1 + for item in message_content + if isinstance(item, dict) and "cachePoint" in item + ) + assert cache_control_count == 1, f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)."