diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index b509548d36..89cafd6d94 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1562,6 +1562,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens_details.text_tokens = calculated_text_tokens ######################################################### + ## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached) if "promptTokensDetails" in usage_metadata: for detail in usage_metadata["promptTokensDetails"]: if detail["modality"] == "AUDIO": @@ -1570,6 +1571,32 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): text_tokens = detail.get("tokenCount", 0) elif detail["modality"] == "IMAGE": image_tokens = detail.get("tokenCount", 0) + + ## Parse cacheTokensDetails (breakdown of cached tokens by modality) + ## When explicit caching is used, Gemini provides this field to show which modalities were cached + cached_text_tokens: Optional[int] = None + cached_audio_tokens: Optional[int] = None + cached_image_tokens: Optional[int] = None + + if "cacheTokensDetails" in usage_metadata: + for detail in usage_metadata["cacheTokensDetails"]: + if detail["modality"] == "AUDIO": + cached_audio_tokens = detail.get("tokenCount", 0) + elif detail["modality"] == "TEXT": + cached_text_tokens = detail.get("tokenCount", 0) + elif detail["modality"] == "IMAGE": + cached_image_tokens = detail.get("tokenCount", 0) + + ## Calculate non-cached tokens by subtracting cached from total (per modality) + ## This is necessary because promptTokensDetails includes both cached and non-cached tokens + ## See: https://github.com/BerriAI/litellm/issues/18750 + if cached_text_tokens is not None and text_tokens is not None: + text_tokens = text_tokens - cached_text_tokens + if cached_audio_tokens is not None and audio_tokens is not None: + audio_tokens = audio_tokens - cached_audio_tokens + if cached_image_tokens is not None and image_tokens is not None: + image_tokens = image_tokens - cached_image_tokens + if "thoughtsTokenCount" in usage_metadata: reasoning_tokens = usage_metadata["thoughtsTokenCount"] # Also add reasoning tokens to response_tokens_details @@ -1577,15 +1604,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens_details = CompletionTokensDetailsWrapper() response_tokens_details.reasoning_tokens = reasoning_tokens - ## adjust 'text_tokens' to subtract cached tokens - if ( - (audio_tokens is None or audio_tokens == 0) - and text_tokens is not None - and text_tokens > 0 - and cached_tokens is not None - ): - text_tokens = text_tokens - cached_tokens - prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cached_tokens, audio_tokens=audio_tokens, @@ -1607,6 +1625,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): completion_tokens=completion_tokens, total_tokens=usage_metadata.get("totalTokenCount", 0), prompt_tokens_details=prompt_tokens_details, + cache_read_input_tokens=cached_tokens, reasoning_tokens=reasoning_tokens, completion_tokens_details=response_tokens_details, ) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 381d91de76..74d6339887 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -260,6 +260,7 @@ class UsageMetadata(TypedDict, total=False): responseTokenCount: int cachedContentTokenCount: int promptTokensDetails: List[PromptTokensDetails] + cacheTokensDetails: List[PromptTokensDetails] thoughtsTokenCount: int responseTokensDetails: List[PromptTokensDetails] candidatesTokensDetails: List[PromptTokensDetails] # Alternative key name used in some responses diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index f94b758f06..0f618aeadb 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1415,3 +1415,92 @@ def test_completion_cost_service_tier_priority(): # Costs should be similar (all using flex) assert abs(cost_from_params - cost_from_usage) < 1e-6, "Costs from params and usage should be similar (both flex)" + + +def test_gemini_cache_tokens_details_no_negative_values(): + """ + Test for Issue #18750: Negative text_tokens with Gemini caching + + When using Gemini with explicit caching, the response includes cacheTokensDetails + which breaks down cached tokens by modality. This test ensures that: + 1. text_tokens is never negative + 2. We correctly subtract cached tokens per modality (not total) + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + # Scenario from issue #18750: Image + text with explicit caching + # Real Gemini response structure when using cached content + completion_response = { + "usageMetadata": { + "promptTokenCount": 9660, + "candidatesTokenCount": 7, + "totalTokenCount": 9667, + "cachedContentTokenCount": 9651, + # Total tokens by modality (includes cached + non-cached) + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 9402}, + {"modality": "IMAGE", "tokenCount": 258} + ], + # Breakdown of cached tokens by modality + "cacheTokensDetails": [ + {"modality": "TEXT", "tokenCount": 9393}, + {"modality": "IMAGE", "tokenCount": 258} + ] + } + } + + usage = VertexGeminiConfig._calculate_usage(completion_response) + + # Text tokens should be non-cached text only: 9402 - 9393 = 9 + assert usage.prompt_tokens_details.text_tokens == 9, \ + f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" + + # Image tokens should be non-cached image only: 258 - 258 = 0 + assert usage.prompt_tokens_details.image_tokens == 0, \ + f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" + + # Total cached should match + assert usage.prompt_tokens_details.cached_tokens == 9651, \ + f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" + + # MOST IMPORTANT: text_tokens should NEVER be negative + assert usage.prompt_tokens_details.text_tokens >= 0, \ + f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" + + print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative") + + +def test_gemini_without_cache_tokens_details(): + """ + Test Gemini response without cacheTokensDetails (implicit caching or no cache) + + When cacheTokensDetails is not present, we should use promptTokensDetails as-is + without subtracting anything. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + completion_response = { + "usageMetadata": { + "promptTokenCount": 264, + "candidatesTokenCount": 15, + "totalTokenCount": 279, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 6}, + {"modality": "IMAGE", "tokenCount": 258} + ] + # No cacheTokensDetails + } + } + + usage = VertexGeminiConfig._calculate_usage(completion_response) + + # Should use promptTokensDetails values directly + assert usage.prompt_tokens_details.text_tokens == 6 + assert usage.prompt_tokens_details.image_tokens == 258 + assert usage.prompt_tokens_details.text_tokens >= 0 + + print("✅ Gemini without cacheTokensDetails works correctly")