mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-11 03:04:02 +00:00
feat(dashscope): preserve cache_control for explicit prompt caching (#25331)
DashScope inherits OpenAIGPTConfig which strips cache_control from messages and tools by default. Override remove_cache_control_flag_from_messages_and_tools() to preserve cache_control, following the same pattern used by ZAI, MiniMax, and Databricks. Verified through 10-round multi-turn conversation tests: - Explicit caching works correctly: cached_tokens grows each round from R4 onwards, with cache_creation_tokens reported on first cache build. - Implicit caching is not affected: models that rely on implicit prefix-matching caching produce identical cached_tokens with and without this change, confirmed by comparing results against both the reverted codebase and direct API calls bypassing litellm. - No errors or regressions observed on any model, including those that do not support explicit caching — the DashScope API silently ignores unrecognized cache_control fields. Fixes #25330 Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,8 @@ Translates from OpenAI's `/v1/chat/completions` to DashScope's `/v1/chat/complet
|
||||
|
||||
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
|
||||
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
@@ -11,6 +13,18 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
||||
class DashScopeChatConfig(OpenAIGPTConfig):
|
||||
def remove_cache_control_flag_from_messages_and_tools(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
tools: Optional[List[ChatCompletionToolParam]] = None,
|
||||
) -> Tuple[List[AllMessageValues], Optional[List[ChatCompletionToolParam]]]:
|
||||
"""
|
||||
Override to preserve cache_control for DashScope.
|
||||
DashScope supports cache_control - don't strip it.
|
||||
"""
|
||||
return messages, tools
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
|
||||
|
||||
@@ -144,3 +144,47 @@ class TestDashScopeConfig:
|
||||
assert transformed_messages[0]["content"][0]["text"] == "Hello"
|
||||
assert transformed_messages[0]["content"][1]["type"] == "text"
|
||||
assert transformed_messages[0]["content"][1]["text"] == "World"
|
||||
|
||||
def test_dashscope_preserves_cache_control_in_messages(self):
|
||||
"""DashScope should NOT strip cache_control from messages."""
|
||||
config = DashScopeChatConfig()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant.",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, world!",
|
||||
},
|
||||
]
|
||||
|
||||
transformed_messages, _ = config.remove_cache_control_flag_from_messages_and_tools(
|
||||
model="dashscope/qwen-turbo", messages=messages
|
||||
)
|
||||
|
||||
assert transformed_messages[0].get("cache_control") == {"type": "ephemeral"}
|
||||
|
||||
def test_dashscope_preserves_cache_control_in_tools(self):
|
||||
"""DashScope should NOT strip cache_control from tools."""
|
||||
config = DashScopeChatConfig()
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
_, transformed_tools = config.remove_cache_control_flag_from_messages_and_tools(
|
||||
model="dashscope/qwen-turbo", messages=[], tools=tools
|
||||
)
|
||||
|
||||
assert transformed_tools[0].get("cache_control") == {"type": "ephemeral"}
|
||||
|
||||
Reference in New Issue
Block a user