From c7a22921dc2f2d4a04e14c095903f8dfcfa063f4 Mon Sep 17 00:00:00 2001 From: shin-bot-litellm Date: Tue, 10 Feb 2026 22:16:41 -0800 Subject: [PATCH] feat: add standard_logging_payload_excluded_fields config option (#20831) Adds a new config option to exclude specific fields from StandardLoggingPayload before any callback receives it. This provides a general approach to control what data is logged across ALL integrations (S3, GCS, Datadog, etc.). ## Changes 1. **litellm/__init__.py**: Added new global setting `standard_logging_payload_excluded_fields: Optional[List[str]] = None` 2. **litellm/integrations/custom_logger.py**: Modified `redact_standard_logging_payload_from_model_call_details()` to: - Remove specified fields entirely from the StandardLoggingPayload - Works alongside existing `turn_off_message_logging` feature - Excluded fields take precedence (removed rather than redacted) 3. **tests/**: Added comprehensive test suite with 17 tests covering: - Single/multiple field exclusion - Interaction with turn_off_message_logging - Original payload immutability - Config loading via setattr (proxy pattern) - Edge cases (empty list, non-existent fields, None standard_logging_object) ## Usage ```yaml litellm_settings: success_callback: ["s3"] standard_logging_payload_excluded_fields: ["response", "messages"] ``` This removes the `response` and `messages` fields from logs before any callback processes them, reducing log size and improving privacy compliance. ## Available Fields The fields match StandardLoggingPayload TypedDict keys including: - messages, response (large payload fields) - metadata, hidden_params, model_parameters - error_str, error_information - And all other StandardLoggingPayload fields Closes the need for per-integration flags like `s3_log_response`. --- litellm/__init__.py | 1 + litellm/integrations/custom_logger.py | 98 +++-- ...tandard_logging_payload_excluded_fields.py | 415 ++++++++++++++++++ 3 files changed, 477 insertions(+), 37 deletions(-) create mode 100644 tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py diff --git a/litellm/__init__.py b/litellm/__init__.py index dacb928e8a..9160297d50 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,6 +175,7 @@ _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # Custo pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 4a341863d4..c244363e38 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -774,15 +774,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self, model_call_details: Dict ) -> Dict: """ - Only redacts messages and responses when self.turn_off_message_logging is True + Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. + This method handles two features: + 1. turn_off_message_logging: When True, redacts messages and responses + 2. standard_logging_payload_excluded_fields: Removes specified fields entirely - By default, self.turn_off_message_logging is False and this does nothing. - - Return a redacted deepcopy of the provided logging payload. + Return a modified copy of the provided logging payload. This is useful for logging payloads that contain sensitive information. """ + import litellm from copy import copy from litellm import Choices, Message, ModelResponse @@ -790,14 +792,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = getattr( self, "turn_off_message_logging", False ) + excluded_fields: Optional[List[str]] = getattr( + litellm, "standard_logging_payload_excluded_fields", None + ) - if turn_off_message_logging is False: + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: return model_call_details # Only make a shallow copy of the top-level dict to avoid deepcopy issues # with complex objects like AuthenticationError that may be present model_call_details_copy = copy(model_call_details) - redacted_str = "redacted-by-litellm" standard_logging_object = model_call_details.get("standard_logging_object") if standard_logging_object is None: return model_call_details_copy @@ -805,39 +810,58 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Make a copy of just the standard_logging_object to avoid modifying the original standard_logging_object_copy = copy(standard_logging_object) - if standard_logging_object_copy.get("messages") is not None: - standard_logging_object_copy["messages"] = [ - Message(content=redacted_str).model_dump() - ] + # Handle excluded fields - remove them entirely from the payload + if excluded_fields: + for field in excluded_fields: + if field in standard_logging_object_copy: + del standard_logging_object_copy[field] - if standard_logging_object_copy.get("response") is not None: - response = standard_logging_object_copy["response"] - # Check if this is a ResponsesAPIResponse (has "output" field) - if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy + # Handle turn_off_message_logging - redact messages and responses (if not already excluded) + if turn_off_message_logging: + redacted_str = "redacted-by-litellm" - response_copy = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy - else: - # Standard ModelResponse format - model_response = ModelResponse( - choices=[Choices(message=Message(content=redacted_str))] - ) - model_response_dict = model_response.model_dump() - standard_logging_object_copy["response"] = model_response_dict + if ( + "messages" not in (excluded_fields or []) + and standard_logging_object_copy.get("messages") is not None + ): + standard_logging_object_copy["messages"] = [ + Message(content=redacted_str).model_dump() + ] + + if ( + "response" not in (excluded_fields or []) + and standard_logging_object_copy.get("response") is not None + ): + response = standard_logging_object_copy["response"] + # Check if this is a ResponsesAPIResponse (has "output" field) + if isinstance(response, dict) and "output" in response: + # Make a copy to avoid modifying the original + from copy import deepcopy + + response_copy = deepcopy(response) + # Redact content in output array + if isinstance(response_copy.get("output"), list): + for output_item in response_copy["output"]: + if ( + isinstance(output_item, dict) + and "content" in output_item + ): + if isinstance(output_item["content"], list): + # Redact text in content items + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + standard_logging_object_copy["response"] = response_copy + else: + # Standard ModelResponse format + model_response = ModelResponse( + choices=[Choices(message=Message(content=redacted_str))] + ) + model_response_dict = model_response.model_dump() + standard_logging_object_copy["response"] = model_response_dict model_call_details_copy["standard_logging_object"] = ( standard_logging_object_copy diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py new file mode 100644 index 0000000000..d3c4ac8056 --- /dev/null +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -0,0 +1,415 @@ +""" +Tests for standard_logging_payload_excluded_fields feature. + +This feature allows users to exclude specific fields from StandardLoggingPayload +before any callback receives it. This is useful for: +- Reducing log sizes (excluding large fields like 'response' or 'messages') +- Privacy compliance (excluding sensitive fields) +- Cost management (less data stored/transmitted) + +Example config: + litellm_settings: + success_callback: ["s3"] + standard_logging_payload_excluded_fields: ["response", "messages"] +""" + +import os +import sys +from copy import deepcopy +from typing import Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload + + +def create_sample_standard_logging_payload() -> Dict: + """Create a sample StandardLoggingPayload for testing.""" + return { + "id": "test-id-123", + "trace_id": "trace-123", + "call_type": "completion", + "stream": False, + "response_cost": 0.001, + "cost_breakdown": None, + "response_cost_failure_debug_info": None, + "status": "success", + "status_fields": {}, + "custom_llm_provider": "openai", + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "startTime": 1234567890.0, + "endTime": 1234567891.0, + "completionStartTime": 1234567890.5, + "response_time": 1.0, + "model_map_information": {}, + "model": "gpt-4", + "model_id": "model-123", + "model_group": None, + "api_base": "https://api.openai.com/v1", + "metadata": {}, + "cache_hit": False, + "cache_key": None, + "saved_cache_cost": 0.0, + "request_tags": [], + "end_user": None, + "requester_ip_address": None, + "user_agent": None, + "messages": [{"role": "user", "content": "Hello, this is sensitive data!"}], + "response": { + "choices": [ + {"message": {"content": "This is a sensitive response!"}} + ] + }, + "error_str": None, + "error_information": None, + "model_parameters": {}, + "hidden_params": {}, + "guardrail_information": None, + "standard_built_in_tools_params": None, + } + + +def create_model_call_details( + standard_logging_payload: Optional[Dict] = None, +) -> Dict: + """Create model_call_details dict with standard_logging_object.""" + if standard_logging_payload is None: + standard_logging_payload = create_sample_standard_logging_payload() + return { + "standard_logging_object": standard_logging_payload, + "other_key": "other_value", + } + + +class TestStandardLoggingPayloadExcludedFields: + """Test suite for standard_logging_payload_excluded_fields feature.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_no_excluded_fields_no_change(self): + """Test that payload is unchanged when no fields are excluded.""" + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_exclude_single_field(self): + """Test excluding a single field (response).""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + + def test_exclude_multiple_fields(self): + """Test excluding multiple fields (response, messages).""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + assert "model_parameters" in result["standard_logging_object"] + + def test_exclude_metadata_field(self): + """Test excluding the metadata field.""" + litellm.standard_logging_payload_excluded_fields = ["metadata"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["metadata"] = {"sensitive_key": "sensitive_value"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "metadata" not in result["standard_logging_object"] + + def test_exclude_hidden_params(self): + """Test excluding hidden_params field.""" + litellm.standard_logging_payload_excluded_fields = ["hidden_params"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["hidden_params"] = {"api_key": "sk-secret-key"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "hidden_params" not in result["standard_logging_object"] + + def test_exclude_nonexistent_field_no_error(self): + """Test that excluding a non-existent field doesn't cause an error.""" + litellm.standard_logging_payload_excluded_fields = [ + "nonexistent_field", + "response", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + # Should not raise an exception + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + + def test_original_payload_not_modified(self): + """Test that the original model_call_details is not modified.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_payload = deepcopy(model_call_details) + + logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Original should still have the fields + assert "response" in model_call_details["standard_logging_object"] + assert "messages" in model_call_details["standard_logging_object"] + assert model_call_details == original_payload + + def test_combined_with_turn_off_message_logging(self): + """Test that excluded_fields works together with turn_off_message_logging.""" + litellm.standard_logging_payload_excluded_fields = ["metadata", "hidden_params"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # excluded_fields should remove these + assert "metadata" not in result["standard_logging_object"] + assert "hidden_params" not in result["standard_logging_object"] + + # turn_off_message_logging should redact these + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + assert ( + result["standard_logging_object"]["response"]["choices"][0]["message"][ + "content" + ] + == redacted_str + ) + + def test_excluded_fields_takes_precedence_over_redaction(self): + """Test that if a field is both excluded and would be redacted, it's excluded.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # response should be excluded (not redacted) + assert "response" not in result["standard_logging_object"] + + # messages should still be redacted + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + + def test_exclude_all_sensitive_fields(self): + """Test excluding all potentially sensitive fields.""" + litellm.standard_logging_payload_excluded_fields = [ + "messages", + "response", + "metadata", + "hidden_params", + "model_parameters", + "error_str", + "error_information", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + standard_obj = result["standard_logging_object"] + + # All sensitive fields should be removed + assert "messages" not in standard_obj + assert "response" not in standard_obj + assert "metadata" not in standard_obj + assert "hidden_params" not in standard_obj + assert "model_parameters" not in standard_obj + assert "error_str" not in standard_obj + assert "error_information" not in standard_obj + + # Non-sensitive fields should remain + assert "id" in standard_obj + assert "model" in standard_obj + assert "response_cost" in standard_obj + assert "total_tokens" in standard_obj + + def test_empty_excluded_fields_list(self): + """Test that an empty list doesn't affect the payload.""" + litellm.standard_logging_payload_excluded_fields = [] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_none_standard_logging_object(self): + """Test handling when standard_logging_object is None.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = {"other_key": "other_value"} + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Should return unchanged when no standard_logging_object + assert result == model_call_details + + +class TestExcludedFieldsIntegration: + """Integration tests for excluded fields with actual callbacks.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def test_custom_callback_receives_filtered_payload(self): + """Test that a custom callback receives the filtered payload.""" + captured_payloads = [] + + class TestCallback(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + captured_payloads.append(kwargs.get("standard_logging_object", {})) + + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + callback = TestCallback() + model_call_details = create_model_call_details() + + # Simulate what litellm_logging.py does + filtered_details = callback.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + callback.log_success_event( + kwargs=filtered_details, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert len(captured_payloads) == 1 + assert "response" not in captured_payloads[0] + assert "messages" not in captured_payloads[0] + assert "model" in captured_payloads[0] + + +class TestExcludedFieldsConfigLoading: + """Test that the config is properly loaded from litellm_settings.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_config_attribute_exists(self): + """Test that the config attribute exists on litellm module.""" + assert hasattr(litellm, "standard_logging_payload_excluded_fields") + + def test_config_default_is_none(self): + """Test that the default value is None.""" + # Reset to ensure we're testing the default + litellm.standard_logging_payload_excluded_fields = None + assert litellm.standard_logging_payload_excluded_fields is None + + def test_config_can_be_set_to_list(self): + """Test that the config can be set to a list.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + assert litellm.standard_logging_payload_excluded_fields == [ + "response", + "messages", + ] + + def test_config_setattr_simulates_proxy_loading(self): + """Test that setattr works as the proxy would use it.""" + # Simulating how proxy_server.py sets litellm_settings + config_value = ["response", "messages", "metadata"] + setattr(litellm, "standard_logging_payload_excluded_fields", config_value) + + assert litellm.standard_logging_payload_excluded_fields == config_value + + # Test it actually works in the logger + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "metadata" not in result["standard_logging_object"]