diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 8b21569546..a7362a9431 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1106,19 +1106,19 @@ class LiteLLMAnthropicMessagesAdapter: # extract usage usage: Usage = getattr(response, "usage") uncached_input_tokens = usage.prompt_tokens or 0 + cached_tokens = 0 if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 uncached_input_tokens -= cached_tokens - + anthropic_usage = AnthropicUsage( input_tokens=uncached_input_tokens, output_tokens=usage.completion_tokens or 0, ) - # Add cache tokens if available (for prompt caching support) if hasattr(usage, "_cache_creation_input_tokens") and usage._cache_creation_input_tokens > 0: anthropic_usage["cache_creation_input_tokens"] = usage._cache_creation_input_tokens - if hasattr(usage, "_cache_read_input_tokens") and usage._cache_read_input_tokens > 0: - anthropic_usage["cache_read_input_tokens"] = usage._cache_read_input_tokens + if cached_tokens > 0: + anthropic_usage["cache_read_input_tokens"] = cached_tokens translated_obj = AnthropicMessagesResponse( id=response.id, @@ -1271,19 +1271,19 @@ class LiteLLMAnthropicMessagesAdapter: litellm_usage_chunk = None if litellm_usage_chunk is not None: uncached_input_tokens = litellm_usage_chunk.prompt_tokens or 0 + cached_tokens = 0 if hasattr(litellm_usage_chunk, "prompt_tokens_details") and litellm_usage_chunk.prompt_tokens_details: cached_tokens = getattr(litellm_usage_chunk.prompt_tokens_details, "cached_tokens", 0) or 0 uncached_input_tokens -= cached_tokens - + usage_delta = UsageDelta( input_tokens=uncached_input_tokens, output_tokens=litellm_usage_chunk.completion_tokens or 0, ) - # Add cache tokens if available (for prompt caching support) if hasattr(litellm_usage_chunk, "_cache_creation_input_tokens") and litellm_usage_chunk._cache_creation_input_tokens > 0: usage_delta["cache_creation_input_tokens"] = litellm_usage_chunk._cache_creation_input_tokens - if hasattr(litellm_usage_chunk, "_cache_read_input_tokens") and litellm_usage_chunk._cache_read_input_tokens > 0: - usage_delta["cache_read_input_tokens"] = litellm_usage_chunk._cache_read_input_tokens + if cached_tokens > 0: + usage_delta["cache_read_input_tokens"] = cached_tokens else: usage_delta = UsageDelta(input_tokens=0, output_tokens=0) return MessageBlockDelta( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 1ef8556233..31615a768d 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,5 +1,6 @@ import hashlib import json +import os import secrets from datetime import datetime from datetime import datetime as dt @@ -10,7 +11,10 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING +from litellm.constants import ( + MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, +) +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -30,6 +34,20 @@ from litellm.types.utils import ( from litellm.utils import get_end_user_id_for_cost_tracking +def _get_max_string_length_prompt_in_db() -> int: + """ + Resolve prompt truncation threshold at runtime so values loaded later via + proxy config environment_variables are honored. + """ + max_length_str = os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB") + if max_length_str is None: + return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB + try: + return int(max_length_str) + except (TypeError, ValueError): + return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB + + def _is_master_key(api_key: str, _master_key: Optional[str]) -> bool: if _master_key is None: return False @@ -609,6 +627,7 @@ def _get_messages_for_spend_logs_payload( def _sanitize_request_body_for_spend_logs_payload( request_body: dict, visited: Optional[set] = None, + max_string_length_prompt_in_db: Optional[int] = None, ) -> dict: """ Recursively sanitize request body to prevent logging large base64 strings or other large values. @@ -618,6 +637,8 @@ def _sanitize_request_body_for_spend_logs_payload( if visited is None: visited = set() + if max_string_length_prompt_in_db is None: + max_string_length_prompt_in_db = _get_max_string_length_prompt_in_db() # Get the object's memory address to track visited objects obj_id = id(request_body) @@ -627,27 +648,29 @@ def _sanitize_request_body_for_spend_logs_payload( def _sanitize_value(value: Any) -> Any: if isinstance(value, dict): - return _sanitize_request_body_for_spend_logs_payload(value, visited) + return _sanitize_request_body_for_spend_logs_payload( + value, visited, max_string_length_prompt_in_db + ) elif isinstance(value, list): return [_sanitize_value(item) for item in value] elif isinstance(value, str): - if len(value) > MAX_STRING_LENGTH_PROMPT_IN_DB: + if len(value) > max_string_length_prompt_in_db: # Keep 35% from beginning and 65% from end (end is usually more important) # This split ensures we keep more context from the end of conversations start_ratio = 0.35 end_ratio = 0.65 # Calculate character distribution - start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * start_ratio) - end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * end_ratio) + start_chars = int(max_string_length_prompt_in_db * start_ratio) + end_chars = int(max_string_length_prompt_in_db * end_ratio) # Ensure we don't exceed the total limit total_keep = start_chars + end_chars - if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB: - end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars + if total_keep > max_string_length_prompt_in_db: + end_chars = max_string_length_prompt_in_db - start_chars # If the string length is less than what we want to keep, just truncate normally - if len(value) <= MAX_STRING_LENGTH_PROMPT_IN_DB: + if len(value) <= max_string_length_prompt_in_db: return value # Calculate how many characters are being skipped diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1ea1374cfb..839d032c43 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1813,6 +1813,51 @@ def test_translate_openai_response_to_anthropic_input_tokens_no_cache(): assert anthropic_response["usage"]["output_tokens"] == 50 +def test_translate_openai_response_to_anthropic_cache_tokens_from_prompt_tokens_details(): + """ + OpenAI/Azure providers set prompt_tokens_details.cached_tokens but not + _cache_read_input_tokens. The adapter should populate cache_read_input_tokens + from prompt_tokens_details.cached_tokens directly. + """ + from litellm.types.utils import PromptTokensDetailsWrapper + + # OpenAI-style usage: only prompt_tokens_details, no cache_read_input_tokens kwarg + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=30 + ), + ) + + response = ModelResponse( + id="test-id", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message( + role="assistant", + content="Test response", + ), + ) + ], + model="gpt-4o-2024-08-06", + usage=usage, + ) + + adapter = LiteLLMAnthropicMessagesAdapter() + anthropic_response = adapter.translate_openai_response_to_anthropic( + response=response, + tool_name_mapping=None, + ) + + assert anthropic_response["usage"]["input_tokens"] == 70 + assert anthropic_response["usage"]["output_tokens"] == 50 + assert anthropic_response["usage"]["cache_read_input_tokens"] == 30 + + # ===================================================================== # Web Search Tool Transformation Tests # ===================================================================== diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 30257cd2da..24f45cc5c9 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -161,6 +161,21 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types(): assert len(sanitized["nested"]["dict"]["key"]) == expected_length +def test_sanitize_request_body_for_spend_logs_payload_uses_runtime_env_override( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + override_max = max(MAX_STRING_LENGTH_PROMPT_IN_DB + 1000, 6000) + test_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + + # Simulate config-loaded env var being set after module import. + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(override_max)) + + sanitized = _sanitize_request_body_for_spend_logs_payload({"text": test_string}) + assert sanitized["text"] == test_string + + def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): # Create a circular reference a: dict[str, Any] = {} @@ -1349,4 +1364,3 @@ def test_get_logging_payload_includes_request_duration_ms(): ) assert payload["request_duration_ms"] == 3000 -