tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-25 22:24:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(adapter): populate cache_read_input_tokens from prompt_tokens_details for OpenAI/Azure (#22090)

Browse Source 
* fix(adapter): populate cache_read_input_tokens from prompt_tokens_details

The Anthropic adapter's translate_openai_response_to_anthropic checked
only the private _cache_read_input_tokens attr (set by Anthropic/DeepSeek)
but not prompt_tokens_details.cached_tokens (set by OpenAI/Azure).

Use prompt_tokens_details.cached_tokens directly — it is already extracted
and is the standard field populated by all providers.

Fixes #22089

* fix(adapter): apply same cache_read_input_tokens fix to streaming path

The streaming path in translate_streaming_openai_response_to_anthropic
had the same bug — relying on _cache_read_input_tokens instead of
prompt_tokens_details.cached_tokens.
This commit is contained in:
Roni Frantchi
2026-02-26 00:10:18 -08:00
committed by GitHub
parent efeaf650aa
commit 475bb94f5a
2 changed files with 53 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
+8 -8
View File
@@ -1106,19 +1106,19 @@ class LiteLLMAnthropicMessagesAdapter:
# extract usage
usage: Usage = getattr(response, "usage")
uncached_input_tokens = usage.prompt_tokens or 0
cached_tokens = 0
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
uncached_input_tokens -= cached_tokens
anthropic_usage = AnthropicUsage(
input_tokens=uncached_input_tokens,
output_tokens=usage.completion_tokens or 0,
)
# Add cache tokens if available (for prompt caching support)
if hasattr(usage, "_cache_creation_input_tokens") and usage._cache_creation_input_tokens > 0:
anthropic_usage["cache_creation_input_tokens"] = usage._cache_creation_input_tokens
if hasattr(usage, "_cache_read_input_tokens") and usage._cache_read_input_tokens > 0:
anthropic_usage["cache_read_input_tokens"] = usage._cache_read_input_tokens
if cached_tokens > 0:
anthropic_usage["cache_read_input_tokens"] = cached_tokens
translated_obj = AnthropicMessagesResponse(
id=response.id,
@@ -1271,19 +1271,19 @@ class LiteLLMAnthropicMessagesAdapter:
litellm_usage_chunk = None
if litellm_usage_chunk is not None:
uncached_input_tokens = litellm_usage_chunk.prompt_tokens or 0
cached_tokens = 0
if hasattr(litellm_usage_chunk, "prompt_tokens_details") and litellm_usage_chunk.prompt_tokens_details:
cached_tokens = getattr(litellm_usage_chunk.prompt_tokens_details, "cached_tokens", 0) or 0
uncached_input_tokens -= cached_tokens
usage_delta = UsageDelta(
input_tokens=uncached_input_tokens,
output_tokens=litellm_usage_chunk.completion_tokens or 0,
)
# Add cache tokens if available (for prompt caching support)
if hasattr(litellm_usage_chunk, "_cache_creation_input_tokens") and litellm_usage_chunk._cache_creation_input_tokens > 0:
usage_delta["cache_creation_input_tokens"] = litellm_usage_chunk._cache_creation_input_tokens
if hasattr(litellm_usage_chunk, "_cache_read_input_tokens") and litellm_usage_chunk._cache_read_input_tokens > 0:
usage_delta["cache_read_input_tokens"] = litellm_usage_chunk._cache_read_input_tokens
if cached_tokens > 0:
usage_delta["cache_read_input_tokens"] = cached_tokens
else:
usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
return MessageBlockDelta(
tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
+45
View File
@@ -1813,6 +1813,51 @@ def test_translate_openai_response_to_anthropic_input_tokens_no_cache():
assert anthropic_response["usage"]["output_tokens"] == 50
def test_translate_openai_response_to_anthropic_cache_tokens_from_prompt_tokens_details():
"""
OpenAI/Azure providers set prompt_tokens_details.cached_tokens but not
_cache_read_input_tokens. The adapter should populate cache_read_input_tokens
from prompt_tokens_details.cached_tokens directly.
"""
from litellm.types.utils import PromptTokensDetailsWrapper
# OpenAI-style usage: only prompt_tokens_details, no cache_read_input_tokens kwarg
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=30
),
)
response = ModelResponse(
id="test-id",
choices=[
Choices(
index=0,
finish_reason="stop",
message=Message(
role="assistant",
content="Test response",
),
)
],
model="gpt-4o-2024-08-06",
usage=usage,
)
adapter = LiteLLMAnthropicMessagesAdapter()
anthropic_response = adapter.translate_openai_response_to_anthropic(
response=response,
tool_name_mapping=None,
)
assert anthropic_response["usage"]["input_tokens"] == 70
assert anthropic_response["usage"]["output_tokens"] == 50
assert anthropic_response["usage"]["cache_read_input_tokens"] == 30
# =====================================================================
# Web Search Tool Transformation Tests
# =====================================================================