From 47ddd0d0bfc04c74c0d4d7ae3884aba4d97c6d6d Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Sat, 14 Mar 2026 15:51:00 -0700 Subject: [PATCH 1/2] fix: redact secrets from proxy log output Add SecretRedactionFilter to scrub API keys, tokens, and credentials from all log records (messages, args, tracebacks, extra fields). - Enable redaction by default; opt out with LITELLM_DISABLE_REDACT_SECRETS=true - Redact patterns: sk-*, Bearer tokens, x-api-key values, base64 creds - Handle JSON formatter exception hooks and percent-style format args - Snapshot dict iteration to avoid RuntimeError during concurrent logging --- litellm/_logging.py | 92 ++++++++- tests/test_litellm/test_secret_redaction.py | 215 ++++++++++++++++++++ 2 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/test_secret_redaction.py diff --git a/litellm/_logging.py b/litellm/_logging.py index fd833f7056..5de9fbb355 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,10 +1,11 @@ import ast import logging import os +import re import sys from datetime import datetime from logging import Formatter -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -15,12 +16,94 @@ if set_verbose is True: logging.warning( "`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs." ) + +_ENABLE_SECRET_REDACTION = os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true" + +_REDACTED = "REDACTED" + + +def _build_secret_patterns() -> re.Pattern: + patterns: List[str] = [ + # AWS access key IDs + r"(?:AKIA|ASIA)[0-9A-Z]{16}", + # AWS secrets / session tokens / access key IDs (key=value) + r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)" + r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}", + # Bearer tokens (OAuth, JWT, etc.) + r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", + # Basic auth headers + r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", + # OpenAI / Anthropic sk- prefixed keys + r"sk-[A-Za-z0-9\-_]{20,}", + # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) + r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", + # x-api-key / api-key header values (handles 'key': 'value' dict repr) + r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", + # Anthropic internal header keys + r"x-ak-[A-Za-z0-9\-_]{20,}", + # Google API keys + r"AIza[0-9A-Za-z\-_]{35}", + # Password / secret params (handles key=value and 'key': 'value') + r"\w*(?:password|passwd|client_secret|secret_key|_secret)" + r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", + # Database connection string credentials (scheme://user:pass@host) + r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", + # Databricks personal access tokens + r"dapi[0-9a-f]{32}", + ] + return re.compile("|".join(patterns), re.IGNORECASE) + + +_SECRET_RE = _build_secret_patterns() + + +def _redact_string(value: str) -> str: + return _SECRET_RE.sub(_REDACTED, value) + + +class SecretRedactionFilter(logging.Filter): + """Scrubs known secret/credential patterns from log records.""" + + _formatter = logging.Formatter() + + def filter(self, record: logging.LogRecord) -> bool: + if not _ENABLE_SECRET_REDACTION: + return True + + try: + record.msg = _redact_string(record.getMessage()) + record.args = None + except Exception: + if isinstance(record.msg, str): + record.msg = _redact_string(record.msg) + + # Redact exception tracebacks + if record.exc_info and record.exc_info[1] is not None: + try: + record.exc_text = _redact_string( + self._formatter.formatException(record.exc_info) + ) + except Exception: + pass + + # Redact extra fields passed via logger.debug("msg", extra={...}) + for key, value in list(record.__dict__.items()): + if key not in _STANDARD_RECORD_ATTRS and isinstance(value, str): + setattr(record, key, _redact_string(value)) + + return True + + +_secret_filter = SecretRedactionFilter() + + json_logs = bool(os.getenv("JSON_LOGS", False)) # Create a handler for the logger (you may need to adapt this based on your needs) log_level = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: str = getattr(logging, log_level.upper()) handler = logging.StreamHandler() handler.setLevel(numeric_level) +handler.addFilter(_secret_filter) def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]: @@ -116,7 +199,7 @@ class JsonFormatter(Formatter): json_record[key] = value if record.exc_info: - json_record["stacktrace"] = self.formatException(record.exc_info) + json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info) return safe_dumps(json_record) @@ -126,6 +209,7 @@ def _setup_json_exception_handlers(formatter): # Create a handler with JSON formatting for exceptions error_handler = logging.StreamHandler() error_handler.setFormatter(formatter) + error_handler.addFilter(_secret_filter) # Setup excepthook for uncaught exceptions def json_excepthook(exc_type, exc_value, exc_traceback): @@ -149,6 +233,7 @@ def _setup_json_exception_handlers(formatter): def async_json_exception_handler(loop, context): exception = context.get("exception") if exception: + exc_type = type(exception) record = logging.LogRecord( name="LiteLLM", level=logging.ERROR, @@ -156,7 +241,7 @@ def _setup_json_exception_handlers(formatter): lineno=0, msg=str(exception), args=(), - exc_info=None, + exc_info=(exc_type, exception, exception.__traceback__), ) error_handler.handle(record) else: @@ -240,6 +325,7 @@ def _initialize_loggers_with_handler(handler: logging.Handler): - Adds a handler to each logger - Prevents bubbling to parent/root (critical to prevent duplicate JSON logs) """ + handler.addFilter(_secret_filter) for lg in _get_loggers_to_initialize(): lg.handlers.clear() # remove any existing handlers lg.addHandler(handler) # add JSON formatter handler diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py new file mode 100644 index 0000000000..821529c111 --- /dev/null +++ b/tests/test_litellm/test_secret_redaction.py @@ -0,0 +1,215 @@ +import logging +import sys +from io import StringIO +from unittest.mock import patch + +import pytest + +from litellm._logging import ( + JsonFormatter, + _redact_string, + _secret_filter, + _setup_json_exception_handlers, + verbose_logger, + verbose_proxy_logger, + verbose_router_logger, +) + +SECRET = "sk-proj-abc123def456ghi789jklmnopqrst" + + +@pytest.fixture(autouse=True) +def _enable_redaction(): + """Ensure secret redaction is on (the default) for all tests in this module.""" + with patch("litellm._logging._ENABLE_SECRET_REDACTION", True): + yield + + +def _capture_logger_output(fn): + """Run fn with all litellm loggers wired to a StringIO buffer, return output.""" + buf = StringIO() + h = logging.StreamHandler(buf) + h.addFilter(_secret_filter) + loggers = [verbose_logger, verbose_proxy_logger, verbose_router_logger] + saved = [(lg, lg.handlers[:], lg.level) for lg in loggers] + for lg in loggers: + lg.handlers.clear() + lg.addHandler(h) + lg.setLevel(logging.DEBUG) + try: + fn() + return buf.getvalue() + finally: + for lg, handlers, level in saved: + lg.handlers.clear() + for old_h in handlers: + lg.addHandler(old_h) + lg.setLevel(level) + + +def test_redact_string_catches_secret_patterns(): + """Core regex patterns redact known secret formats.""" + cases = [ + "Bearer eyJhbGciOiJSUzI1NiJ9.payload.sig", + "api_key=a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", + "password=supersecretpassword123", + "postgresql://admin:s3cretpass@db.example.com:5432/mydb", + SECRET, + ] + for secret in cases: + result = _redact_string("msg: " + secret) + assert secret not in result, f"{secret!r} was not redacted" + assert "REDACTED" in result + + normal = "Loaded model gpt-4 with 3 replicas on us-east-1" + assert _redact_string(normal) == normal + + +def test_filter_redacts_secrets_in_logger_output(): + def log_messages(): + verbose_logger.debug("Key: " + SECRET) + verbose_logger.debug("Normal message with no secrets") + + output = _capture_logger_output(log_messages) + assert SECRET not in output + assert "REDACTED" in output + assert "Normal message with no secrets" in output + + +def test_filter_redacts_percent_style_args(): + """Secrets passed as %-style args should be redacted.""" + + def log_messages(): + verbose_logger.debug("key=%s region=%s", SECRET, "us-east-1") + + output = _capture_logger_output(log_messages) + assert SECRET not in output + assert "us-east-1" in output + + +def test_filter_redacts_non_string_args(): + """Secrets inside dicts/lists passed as %-style args should be redacted.""" + + def log_messages(): + verbose_logger.debug("Config: %s", {"nested": {"key": SECRET}}) + verbose_logger.debug("Keys: %s", [SECRET]) + + output = _capture_logger_output(log_messages) + assert SECRET not in output + assert "REDACTED" in output + + +def test_filter_redacts_exception_tracebacks(): + """Secrets embedded in exception messages must be redacted in tracebacks.""" + + def log_messages(): + try: + raise ValueError(f"Auth failed with key {SECRET}") + except ValueError: + verbose_logger.exception("Something went wrong") + + output = _capture_logger_output(log_messages) + assert SECRET not in output + assert "REDACTED" in output + assert "Something went wrong" in output + + +def test_filter_redacts_extra_fields(): + """Secrets passed via extra={...} must be redacted on the record.""" + record = logging.LogRecord( + name="test", + level=logging.DEBUG, + pathname="", + lineno=0, + msg="request completed", + args=(), + exc_info=None, + ) + record.api_key = SECRET + record.region = "us-east-1" + + _secret_filter.filter(record) + + assert SECRET not in record.api_key + assert "REDACTED" in record.api_key + assert record.region == "us-east-1" + + +def test_disable_redaction_passes_secrets_through(): + """When LITELLM_DISABLE_REDACT_SECRETS=true, secrets pass through.""" + with patch("litellm._logging._ENABLE_SECRET_REDACTION", False): + record = logging.LogRecord( + name="test", + level=logging.DEBUG, + pathname="", + lineno=0, + msg="key=" + SECRET, + args=(), + exc_info=None, + ) + _secret_filter.filter(record) + assert "sk-proj-" in record.msg + + +def test_x_api_key_regex_does_not_consume_json_delimiters(): + """x-api-key pattern must stop before closing quotes/braces so JSON stays valid.""" + # Simulates a JSON log line containing an x-api-key header value + json_line = '{"headers": {"x-api-key": "secret123"}, "status": 200}' + result = _redact_string(json_line) + # The secret value should be redacted + assert "secret123" not in result + assert "REDACTED" in result + # Closing delimiter must survive so the line is still valid-ish JSON + assert '"status": 200' in result + assert "}" in result + + +def test_json_excepthook_redacts_secrets(): + """Unhandled exceptions in JSON mode must have secrets redacted.""" + buf = StringIO() + h = logging.StreamHandler(buf) + h.setFormatter(JsonFormatter()) + h.addFilter(_secret_filter) + + # Capture what the excepthook would emit + record = logging.LogRecord( + name="LiteLLM", + level=logging.ERROR, + pathname="", + lineno=0, + msg=f"Connection failed with key {SECRET}", + args=(), + exc_info=None, + ) + # Simulate the filter + formatter pipeline + _secret_filter.filter(record) + output = h.formatter.format(record) + assert SECRET not in output + assert "REDACTED" in output + + +def test_json_excepthook_redacts_traceback_secrets(): + """Unhandled exception tracebacks in JSON mode must have secrets redacted.""" + buf = StringIO() + h = logging.StreamHandler(buf) + h.setFormatter(JsonFormatter()) + h.addFilter(_secret_filter) + + try: + raise RuntimeError(f"Failed to auth with {SECRET}") + except RuntimeError: + exc_info = sys.exc_info() + + record = logging.LogRecord( + name="LiteLLM", + level=logging.ERROR, + pathname="", + lineno=0, + msg=str(exc_info[1]), + args=(), + exc_info=exc_info, + ) + _secret_filter.filter(record) + output = h.formatter.format(record) + assert SECRET not in output + assert "REDACTED" in output From e45c82aea070fe40f725fbeb40eb83493b80f310 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Sat, 14 Mar 2026 15:59:52 -0700 Subject: [PATCH 2/2] docs: add LITELLM_DISABLE_REDACT_SECRETS to environment variable reference --- docs/my-website/docs/proxy/config_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4f25e6c109..a0e404e3a1 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -778,6 +778,7 @@ router_settings: | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659) +| LITELLM_DISABLE_REDACT_SECRETS | When set to "true", disables automatic redaction of secrets (API keys, tokens, credentials) from proxy log output. Secret redaction is enabled by default. | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.