Merge pull request #20397 from ryan-crabbe/fix/openai-prompt-cache-params

fix: add prompt_cache_key and prompt_cache_retention support for OpenAI

docs/my-website/docs/completion/prompt_caching.md

@@ -63,7 +63,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -77,7 +76,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -112,16 +110,16 @@ model_list:
         api_key: os.environ/OPENAI_API_KEY
 ```
 
-2. Start proxy 
+2. Start proxy
 
 ```bash
 litellm --config /path/to/config.yaml
 ```
 
-3. Test it! 
+3. Test it!
 
 ```python
-from openai import OpenAI 
+from openai import OpenAI
 import os
 
 client = OpenAI(
@@ -144,7 +142,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -158,7 +155,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
 </TabItem>
 </Tabs>
 
+### OpenAI `prompt_cache_key` and `prompt_cache_retention`
+
+OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
+
+OpenAI also supports two optional parameters for more control over caching behavior:
+
+- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
+- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = ""
+
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    prompt_cache_key="legal-doc-analysis",
+    prompt_cache_retention="24h",
+)
+print(response.usage)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",
+    base_url="LITELLM_PROXY_BASE",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    extra_body={
+        "prompt_cache_key": "legal-doc-analysis",
+        "prompt_cache_retention": "24h",
+    },
+)
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Anthropic Example 
 
 Anthropic charges for cache writes. 

litellm/llms/openai/chat/gpt_transformation.py

@@ -162,6 +162,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             "service_tier",
             "safety_identifier",
             "prompt_cache_key",
+            "prompt_cache_retention",
             "store",
         ]  # works across all models
 

litellm/types/llms/openai.py

@@ -1125,6 +1125,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
     prompt: Optional[PromptObject]
     max_tool_calls: Optional[int]
     prompt_cache_key: Optional[str]
+    prompt_cache_retention: Optional[str]
     stream_options: Optional[dict]
     top_logprobs: Optional[int]
     partial_images: Optional[

tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py

@@ -207,10 +207,10 @@ class TestOpenAIChatCompletionStreamingHandler:
     def test_chunk_parser_maps_reasoning_to_reasoning_content(self):
         """
         Test that chunk_parser maps 'reasoning' field to 'reasoning_content'.
-        
+
         Some OpenAI-compatible providers (e.g., GLM-5, hosted_vllm) return
         delta.reasoning, but LiteLLM expects delta.reasoning_content.
-        
+
         Regression test for: Streaming responses with delta.reasoning field
         coming back empty when using openai/ or hosted_vllm/ providers.
         """
@@ -293,3 +293,34 @@ class TestPromptCacheKeyIntegration:
             prompt_cache_key="test-cache-key-123",
         )
         assert optional_params.get("prompt_cache_key") == "test-cache-key-123"
+
+
+class TestPromptCacheParams:
+    """Tests for prompt_cache_key and prompt_cache_retention support."""
+
+    def setup_method(self):
+        self.config = OpenAIGPTConfig()
+
+    def test_prompt_cache_key_in_supported_params(self):
+        """Test that prompt_cache_key is in supported params for OpenAI models."""
+        supported_params = self.config.get_supported_openai_params("gpt-4o")
+        assert "prompt_cache_key" in supported_params
+
+    def test_prompt_cache_retention_in_supported_params(self):
+        """Test that prompt_cache_retention is in supported params for OpenAI models."""
+        supported_params = self.config.get_supported_openai_params("gpt-4o")
+        assert "prompt_cache_retention" in supported_params
+
+    def test_prompt_cache_params_passed_through(self):
+        """Test that prompt_cache_key and prompt_cache_retention are passed through by map_openai_params."""
+        optional_params = self.config.map_openai_params(
+            non_default_params={
+                "prompt_cache_key": "my-cache-key",
+                "prompt_cache_retention": "24h",
+            },
+            optional_params={},
+            model="gpt-4o",
+            drop_params=False,
+        )
+        assert optional_params.get("prompt_cache_key") == "my-cache-key"
+        assert optional_params.get("prompt_cache_retention") == "24h"