Merge pull request #22166 from BerriAI/litellm_oss_staging_02_26_2026

Litellm oss staging 02 26 2026

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -1106,19 +1106,19 @@ class LiteLLMAnthropicMessagesAdapter:
         # extract usage
         usage: Usage = getattr(response, "usage")
         uncached_input_tokens = usage.prompt_tokens or 0
+        cached_tokens = 0
         if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
             cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
             uncached_input_tokens -= cached_tokens
-        
+
         anthropic_usage = AnthropicUsage(
             input_tokens=uncached_input_tokens,
             output_tokens=usage.completion_tokens or 0,
         )
-        # Add cache tokens if available (for prompt caching support)
         if hasattr(usage, "_cache_creation_input_tokens") and usage._cache_creation_input_tokens > 0:
             anthropic_usage["cache_creation_input_tokens"] = usage._cache_creation_input_tokens
-        if hasattr(usage, "_cache_read_input_tokens") and usage._cache_read_input_tokens > 0:
-            anthropic_usage["cache_read_input_tokens"] = usage._cache_read_input_tokens
+        if cached_tokens > 0:
+            anthropic_usage["cache_read_input_tokens"] = cached_tokens
 
         translated_obj = AnthropicMessagesResponse(
             id=response.id,
@@ -1271,19 +1271,19 @@ class LiteLLMAnthropicMessagesAdapter:
                 litellm_usage_chunk = None
             if litellm_usage_chunk is not None:
                 uncached_input_tokens = litellm_usage_chunk.prompt_tokens or 0
+                cached_tokens = 0
                 if hasattr(litellm_usage_chunk, "prompt_tokens_details") and litellm_usage_chunk.prompt_tokens_details:
                     cached_tokens = getattr(litellm_usage_chunk.prompt_tokens_details, "cached_tokens", 0) or 0
                     uncached_input_tokens -= cached_tokens
-                
+
                 usage_delta = UsageDelta(
                     input_tokens=uncached_input_tokens,
                     output_tokens=litellm_usage_chunk.completion_tokens or 0,
                 )
-                # Add cache tokens if available (for prompt caching support)
                 if hasattr(litellm_usage_chunk, "_cache_creation_input_tokens") and litellm_usage_chunk._cache_creation_input_tokens > 0:
                     usage_delta["cache_creation_input_tokens"] = litellm_usage_chunk._cache_creation_input_tokens
-                if hasattr(litellm_usage_chunk, "_cache_read_input_tokens") and litellm_usage_chunk._cache_read_input_tokens > 0:
-                    usage_delta["cache_read_input_tokens"] = litellm_usage_chunk._cache_read_input_tokens
+                if cached_tokens > 0:
+                    usage_delta["cache_read_input_tokens"] = cached_tokens
             else:
                 usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
             return MessageBlockDelta(

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import os
 import secrets
 from datetime import datetime
 from datetime import datetime as dt
@@ -10,7 +11,10 @@ from pydantic import BaseModel
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING
+from litellm.constants import (
+    MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
+)
+from litellm.constants import REDACTED_BY_LITELM_STRING
 from litellm.litellm_core_utils.core_helpers import (
     get_litellm_metadata_from_kwargs,
     reconstruct_model_name,
@@ -30,6 +34,20 @@ from litellm.types.utils import (
 from litellm.utils import get_end_user_id_for_cost_tracking
 
 
+def _get_max_string_length_prompt_in_db() -> int:
+    """
+    Resolve prompt truncation threshold at runtime so values loaded later via
+    proxy config environment_variables are honored.
+    """
+    max_length_str = os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB")
+    if max_length_str is None:
+        return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
+    try:
+        return int(max_length_str)
+    except (TypeError, ValueError):
+        return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
+
+
 def _is_master_key(api_key: str, _master_key: Optional[str]) -> bool:
     if _master_key is None:
         return False
@@ -609,6 +627,7 @@ def _get_messages_for_spend_logs_payload(
 def _sanitize_request_body_for_spend_logs_payload(
     request_body: dict,
     visited: Optional[set] = None,
+    max_string_length_prompt_in_db: Optional[int] = None,
 ) -> dict:
     """
     Recursively sanitize request body to prevent logging large base64 strings or other large values.
@@ -618,6 +637,8 @@ def _sanitize_request_body_for_spend_logs_payload(
 
     if visited is None:
         visited = set()
+    if max_string_length_prompt_in_db is None:
+        max_string_length_prompt_in_db = _get_max_string_length_prompt_in_db()
 
     # Get the object's memory address to track visited objects
     obj_id = id(request_body)
@@ -627,27 +648,29 @@ def _sanitize_request_body_for_spend_logs_payload(
 
     def _sanitize_value(value: Any) -> Any:
         if isinstance(value, dict):
-            return _sanitize_request_body_for_spend_logs_payload(value, visited)
+            return _sanitize_request_body_for_spend_logs_payload(
+                value, visited, max_string_length_prompt_in_db
+            )
         elif isinstance(value, list):
             return [_sanitize_value(item) for item in value]
         elif isinstance(value, str):
-            if len(value) > MAX_STRING_LENGTH_PROMPT_IN_DB:
+            if len(value) > max_string_length_prompt_in_db:
                 # Keep 35% from beginning and 65% from end (end is usually more important)
                 # This split ensures we keep more context from the end of conversations
                 start_ratio = 0.35
                 end_ratio = 0.65
 
                 # Calculate character distribution
-                start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * start_ratio)
-                end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * end_ratio)
+                start_chars = int(max_string_length_prompt_in_db * start_ratio)
+                end_chars = int(max_string_length_prompt_in_db * end_ratio)
 
                 # Ensure we don't exceed the total limit
                 total_keep = start_chars + end_chars
-                if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
-                    end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
+                if total_keep > max_string_length_prompt_in_db:
+                    end_chars = max_string_length_prompt_in_db - start_chars
 
                 # If the string length is less than what we want to keep, just truncate normally
-                if len(value) <= MAX_STRING_LENGTH_PROMPT_IN_DB:
+                if len(value) <= max_string_length_prompt_in_db:
                     return value
 
                 # Calculate how many characters are being skipped

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -1813,6 +1813,51 @@ def test_translate_openai_response_to_anthropic_input_tokens_no_cache():
     assert anthropic_response["usage"]["output_tokens"] == 50
 
 
+def test_translate_openai_response_to_anthropic_cache_tokens_from_prompt_tokens_details():
+    """
+    OpenAI/Azure providers set prompt_tokens_details.cached_tokens but not
+    _cache_read_input_tokens.  The adapter should populate cache_read_input_tokens
+    from prompt_tokens_details.cached_tokens directly.
+    """
+    from litellm.types.utils import PromptTokensDetailsWrapper
+
+    # OpenAI-style usage: only prompt_tokens_details, no cache_read_input_tokens kwarg
+    usage = Usage(
+        prompt_tokens=100,
+        completion_tokens=50,
+        total_tokens=150,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=30
+        ),
+    )
+
+    response = ModelResponse(
+        id="test-id",
+        choices=[
+            Choices(
+                index=0,
+                finish_reason="stop",
+                message=Message(
+                    role="assistant",
+                    content="Test response",
+                ),
+            )
+        ],
+        model="gpt-4o-2024-08-06",
+        usage=usage,
+    )
+
+    adapter = LiteLLMAnthropicMessagesAdapter()
+    anthropic_response = adapter.translate_openai_response_to_anthropic(
+        response=response,
+        tool_name_mapping=None,
+    )
+
+    assert anthropic_response["usage"]["input_tokens"] == 70
+    assert anthropic_response["usage"]["output_tokens"] == 50
+    assert anthropic_response["usage"]["cache_read_input_tokens"] == 30
+
+
 # =====================================================================
 # Web Search Tool Transformation Tests
 # =====================================================================

tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py

@@ -161,6 +161,21 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types():
     assert len(sanitized["nested"]["dict"]["key"]) == expected_length
 
 
+def test_sanitize_request_body_for_spend_logs_payload_uses_runtime_env_override(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
+
+    override_max = max(MAX_STRING_LENGTH_PROMPT_IN_DB + 1000, 6000)
+    test_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
+
+    # Simulate config-loaded env var being set after module import.
+    monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(override_max))
+
+    sanitized = _sanitize_request_body_for_spend_logs_payload({"text": test_string})
+    assert sanitized["text"] == test_string
+
+
 def test_sanitize_request_body_for_spend_logs_payload_circular_reference():
     # Create a circular reference
     a: dict[str, Any] = {}
@@ -1349,4 +1364,3 @@ def test_get_logging_payload_includes_request_duration_ms():
         )
 
     assert payload["request_duration_ms"] == 3000
-