diff --git a/docs/my-website/docs/completion/prompt_caching.md b/docs/my-website/docs/completion/prompt_caching.md index 630c9e58d2..dca5f5c0cf 100644 --- a/docs/my-website/docs/completion/prompt_caching.md +++ b/docs/my-website/docs/completion/prompt_caching.md @@ -63,7 +63,6 @@ for _ in range(2): } ], }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. { "role": "user", "content": [ @@ -77,7 +76,6 @@ for _ in range(2): "role": "assistant", "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", }, - # The final turn is marked with cache-control, for continuing in followups. { "role": "user", "content": [ @@ -112,16 +110,16 @@ model_list: api_key: os.environ/OPENAI_API_KEY ``` -2. Start proxy +2. Start proxy ```bash litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```python -from openai import OpenAI +from openai import OpenAI import os client = OpenAI( @@ -144,7 +142,6 @@ for _ in range(2): } ], }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. { "role": "user", "content": [ @@ -158,7 +155,6 @@ for _ in range(2): "role": "assistant", "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", }, - # The final turn is marked with cache-control, for continuing in followups. { "role": "user", "content": [ @@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0 +### OpenAI `prompt_cache_key` and `prompt_cache_retention` + +OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching. + +OpenAI also supports two optional parameters for more control over caching behavior: + +- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit. +- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage. + + + + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "" + +response = completion( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": "You are an AI assistant tasked with analyzing legal documents. " + + "Here is the full text of a complex legal agreement " * 400, + }, + { + "role": "user", + "content": "What are the key terms and conditions?", + }, + ], + prompt_cache_key="legal-doc-analysis", + prompt_cache_retention="24h", +) +print(response.usage) +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="LITELLM_PROXY_KEY", + base_url="LITELLM_PROXY_BASE", +) + +response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": "You are an AI assistant tasked with analyzing legal documents. " + + "Here is the full text of a complex legal agreement " * 400, + }, + { + "role": "user", + "content": "What are the key terms and conditions?", + }, + ], + extra_body={ + "prompt_cache_key": "legal-doc-analysis", + "prompt_cache_retention": "24h", + }, +) +print(response.usage) +``` + + + + ### Anthropic Example Anthropic charges for cache writes. diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index aa8471a597..ab102a6967 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -162,6 +162,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "service_tier", "safety_identifier", "prompt_cache_key", + "prompt_cache_retention", "store", ] # works across all models diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4ab81f8fd5..15e8d1be93 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1125,6 +1125,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): prompt: Optional[PromptObject] max_tool_calls: Optional[int] prompt_cache_key: Optional[str] + prompt_cache_retention: Optional[str] stream_options: Optional[dict] top_logprobs: Optional[int] partial_images: Optional[ diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 37959f7408..39ff0a4f4d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -207,10 +207,10 @@ class TestOpenAIChatCompletionStreamingHandler: def test_chunk_parser_maps_reasoning_to_reasoning_content(self): """ Test that chunk_parser maps 'reasoning' field to 'reasoning_content'. - + Some OpenAI-compatible providers (e.g., GLM-5, hosted_vllm) return delta.reasoning, but LiteLLM expects delta.reasoning_content. - + Regression test for: Streaming responses with delta.reasoning field coming back empty when using openai/ or hosted_vllm/ providers. """ @@ -293,3 +293,34 @@ class TestPromptCacheKeyIntegration: prompt_cache_key="test-cache-key-123", ) assert optional_params.get("prompt_cache_key") == "test-cache-key-123" + + +class TestPromptCacheParams: + """Tests for prompt_cache_key and prompt_cache_retention support.""" + + def setup_method(self): + self.config = OpenAIGPTConfig() + + def test_prompt_cache_key_in_supported_params(self): + """Test that prompt_cache_key is in supported params for OpenAI models.""" + supported_params = self.config.get_supported_openai_params("gpt-4o") + assert "prompt_cache_key" in supported_params + + def test_prompt_cache_retention_in_supported_params(self): + """Test that prompt_cache_retention is in supported params for OpenAI models.""" + supported_params = self.config.get_supported_openai_params("gpt-4o") + assert "prompt_cache_retention" in supported_params + + def test_prompt_cache_params_passed_through(self): + """Test that prompt_cache_key and prompt_cache_retention are passed through by map_openai_params.""" + optional_params = self.config.map_openai_params( + non_default_params={ + "prompt_cache_key": "my-cache-key", + "prompt_cache_retention": "24h", + }, + optional_params={}, + model="gpt-4o", + drop_params=False, + ) + assert optional_params.get("prompt_cache_key") == "my-cache-key" + assert optional_params.get("prompt_cache_retention") == "24h"