Merge pull request #23668 from BerriAI/litellm_fix_redaction_filter_logging_v2

fix: redact secrets from proxy log output

docs/my-website/docs/proxy/config_settings.md

@@ -778,6 +778,7 @@ router_settings:
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
 | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
+| LITELLM_DISABLE_REDACT_SECRETS | When set to "true", disables automatic redaction of secrets (API keys, tokens, credentials) from proxy log output. Secret redaction is enabled by default.
 | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
 | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
 | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.

litellm/_logging.py

@@ -1,10 +1,11 @@
 import ast
 import logging
 import os
+import re
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -15,12 +16,94 @@ if set_verbose is True:
     logging.warning(
         "`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."
     )
+
+_ENABLE_SECRET_REDACTION = os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
+
+_REDACTED = "REDACTED"
+
+
+def _build_secret_patterns() -> re.Pattern:
+    patterns: List[str] = [
+        # AWS access key IDs
+        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
+        # AWS secrets / session tokens / access key IDs (key=value)
+        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
+        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
+        # Bearer tokens (OAuth, JWT, etc.)
+        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
+        # Basic auth headers
+        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
+        # OpenAI / Anthropic sk- prefixed keys
+        r"sk-[A-Za-z0-9\-_]{20,}",
+        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
+        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
+        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
+        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Anthropic internal header keys
+        r"x-ak-[A-Za-z0-9\-_]{20,}",
+        # Google API keys
+        r"AIza[0-9A-Za-z\-_]{35}",
+        # Password / secret params (handles key=value and 'key': 'value')
+        r"\w*(?:password|passwd|client_secret|secret_key|_secret)"
+        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Database connection string credentials (scheme://user:pass@host)
+        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
+        # Databricks personal access tokens
+        r"dapi[0-9a-f]{32}",
+    ]
+    return re.compile("|".join(patterns), re.IGNORECASE)
+
+
+_SECRET_RE = _build_secret_patterns()
+
+
+def _redact_string(value: str) -> str:
+    return _SECRET_RE.sub(_REDACTED, value)
+
+
+class SecretRedactionFilter(logging.Filter):
+    """Scrubs known secret/credential patterns from log records."""
+
+    _formatter = logging.Formatter()
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not _ENABLE_SECRET_REDACTION:
+            return True
+
+        try:
+            record.msg = _redact_string(record.getMessage())
+            record.args = None
+        except Exception:
+            if isinstance(record.msg, str):
+                record.msg = _redact_string(record.msg)
+
+        # Redact exception tracebacks
+        if record.exc_info and record.exc_info[1] is not None:
+            try:
+                record.exc_text = _redact_string(
+                    self._formatter.formatException(record.exc_info)
+                )
+            except Exception:
+                pass
+
+        # Redact extra fields passed via logger.debug("msg", extra={...})
+        for key, value in list(record.__dict__.items()):
+            if key not in _STANDARD_RECORD_ATTRS and isinstance(value, str):
+                setattr(record, key, _redact_string(value))
+
+        return True
+
+
+_secret_filter = SecretRedactionFilter()
+
+
 json_logs = bool(os.getenv("JSON_LOGS", False))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: str = getattr(logging, log_level.upper())
 handler = logging.StreamHandler()
 handler.setLevel(numeric_level)
+handler.addFilter(_secret_filter)
 
 
 def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]:
@@ -116,7 +199,7 @@ class JsonFormatter(Formatter):
                 json_record[key] = value
 
         if record.exc_info:
-            json_record["stacktrace"] = self.formatException(record.exc_info)
+            json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
 
         return safe_dumps(json_record)
 
@@ -126,6 +209,7 @@ def _setup_json_exception_handlers(formatter):
     # Create a handler with JSON formatting for exceptions
     error_handler = logging.StreamHandler()
     error_handler.setFormatter(formatter)
+    error_handler.addFilter(_secret_filter)
 
     # Setup excepthook for uncaught exceptions
     def json_excepthook(exc_type, exc_value, exc_traceback):
@@ -149,6 +233,7 @@ def _setup_json_exception_handlers(formatter):
         def async_json_exception_handler(loop, context):
             exception = context.get("exception")
             if exception:
+                exc_type = type(exception)
                 record = logging.LogRecord(
                     name="LiteLLM",
                     level=logging.ERROR,
@@ -156,7 +241,7 @@ def _setup_json_exception_handlers(formatter):
                     lineno=0,
                     msg=str(exception),
                     args=(),
-                    exc_info=None,
+                    exc_info=(exc_type, exception, exception.__traceback__),
                 )
                 error_handler.handle(record)
             else:
@@ -240,6 +325,7 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
     - Adds a handler to each logger
     - Prevents bubbling to parent/root (critical to prevent duplicate JSON logs)
     """
+    handler.addFilter(_secret_filter)
     for lg in _get_loggers_to_initialize():
         lg.handlers.clear()  # remove any existing handlers
         lg.addHandler(handler)  # add JSON formatter handler

tests/test_litellm/test_secret_redaction.py

@@ -0,0 +1,215 @@
+import logging
+import sys
+from io import StringIO
+from unittest.mock import patch
+
+import pytest
+
+from litellm._logging import (
+    JsonFormatter,
+    _redact_string,
+    _secret_filter,
+    _setup_json_exception_handlers,
+    verbose_logger,
+    verbose_proxy_logger,
+    verbose_router_logger,
+)
+
+SECRET = "sk-proj-abc123def456ghi789jklmnopqrst"
+
+
+@pytest.fixture(autouse=True)
+def _enable_redaction():
+    """Ensure secret redaction is on (the default) for all tests in this module."""
+    with patch("litellm._logging._ENABLE_SECRET_REDACTION", True):
+        yield
+
+
+def _capture_logger_output(fn):
+    """Run fn with all litellm loggers wired to a StringIO buffer, return output."""
+    buf = StringIO()
+    h = logging.StreamHandler(buf)
+    h.addFilter(_secret_filter)
+    loggers = [verbose_logger, verbose_proxy_logger, verbose_router_logger]
+    saved = [(lg, lg.handlers[:], lg.level) for lg in loggers]
+    for lg in loggers:
+        lg.handlers.clear()
+        lg.addHandler(h)
+        lg.setLevel(logging.DEBUG)
+    try:
+        fn()
+        return buf.getvalue()
+    finally:
+        for lg, handlers, level in saved:
+            lg.handlers.clear()
+            for old_h in handlers:
+                lg.addHandler(old_h)
+            lg.setLevel(level)
+
+
+def test_redact_string_catches_secret_patterns():
+    """Core regex patterns redact known secret formats."""
+    cases = [
+        "Bearer eyJhbGciOiJSUzI1NiJ9.payload.sig",
+        "api_key=a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6",
+        "password=supersecretpassword123",
+        "postgresql://admin:s3cretpass@db.example.com:5432/mydb",
+        SECRET,
+    ]
+    for secret in cases:
+        result = _redact_string("msg: " + secret)
+        assert secret not in result, f"{secret!r} was not redacted"
+        assert "REDACTED" in result
+
+    normal = "Loaded model gpt-4 with 3 replicas on us-east-1"
+    assert _redact_string(normal) == normal
+
+
+def test_filter_redacts_secrets_in_logger_output():
+    def log_messages():
+        verbose_logger.debug("Key: " + SECRET)
+        verbose_logger.debug("Normal message with no secrets")
+
+    output = _capture_logger_output(log_messages)
+    assert SECRET not in output
+    assert "REDACTED" in output
+    assert "Normal message with no secrets" in output
+
+
+def test_filter_redacts_percent_style_args():
+    """Secrets passed as %-style args should be redacted."""
+
+    def log_messages():
+        verbose_logger.debug("key=%s region=%s", SECRET, "us-east-1")
+
+    output = _capture_logger_output(log_messages)
+    assert SECRET not in output
+    assert "us-east-1" in output
+
+
+def test_filter_redacts_non_string_args():
+    """Secrets inside dicts/lists passed as %-style args should be redacted."""
+
+    def log_messages():
+        verbose_logger.debug("Config: %s", {"nested": {"key": SECRET}})
+        verbose_logger.debug("Keys: %s", [SECRET])
+
+    output = _capture_logger_output(log_messages)
+    assert SECRET not in output
+    assert "REDACTED" in output
+
+
+def test_filter_redacts_exception_tracebacks():
+    """Secrets embedded in exception messages must be redacted in tracebacks."""
+
+    def log_messages():
+        try:
+            raise ValueError(f"Auth failed with key {SECRET}")
+        except ValueError:
+            verbose_logger.exception("Something went wrong")
+
+    output = _capture_logger_output(log_messages)
+    assert SECRET not in output
+    assert "REDACTED" in output
+    assert "Something went wrong" in output
+
+
+def test_filter_redacts_extra_fields():
+    """Secrets passed via extra={...} must be redacted on the record."""
+    record = logging.LogRecord(
+        name="test",
+        level=logging.DEBUG,
+        pathname="",
+        lineno=0,
+        msg="request completed",
+        args=(),
+        exc_info=None,
+    )
+    record.api_key = SECRET
+    record.region = "us-east-1"
+
+    _secret_filter.filter(record)
+
+    assert SECRET not in record.api_key
+    assert "REDACTED" in record.api_key
+    assert record.region == "us-east-1"
+
+
+def test_disable_redaction_passes_secrets_through():
+    """When LITELLM_DISABLE_REDACT_SECRETS=true, secrets pass through."""
+    with patch("litellm._logging._ENABLE_SECRET_REDACTION", False):
+        record = logging.LogRecord(
+            name="test",
+            level=logging.DEBUG,
+            pathname="",
+            lineno=0,
+            msg="key=" + SECRET,
+            args=(),
+            exc_info=None,
+        )
+        _secret_filter.filter(record)
+        assert "sk-proj-" in record.msg
+
+
+def test_x_api_key_regex_does_not_consume_json_delimiters():
+    """x-api-key pattern must stop before closing quotes/braces so JSON stays valid."""
+    # Simulates a JSON log line containing an x-api-key header value
+    json_line = '{"headers": {"x-api-key": "secret123"}, "status": 200}'
+    result = _redact_string(json_line)
+    # The secret value should be redacted
+    assert "secret123" not in result
+    assert "REDACTED" in result
+    # Closing delimiter must survive so the line is still valid-ish JSON
+    assert '"status": 200' in result
+    assert "}" in result
+
+
+def test_json_excepthook_redacts_secrets():
+    """Unhandled exceptions in JSON mode must have secrets redacted."""
+    buf = StringIO()
+    h = logging.StreamHandler(buf)
+    h.setFormatter(JsonFormatter())
+    h.addFilter(_secret_filter)
+
+    # Capture what the excepthook would emit
+    record = logging.LogRecord(
+        name="LiteLLM",
+        level=logging.ERROR,
+        pathname="",
+        lineno=0,
+        msg=f"Connection failed with key {SECRET}",
+        args=(),
+        exc_info=None,
+    )
+    # Simulate the filter + formatter pipeline
+    _secret_filter.filter(record)
+    output = h.formatter.format(record)
+    assert SECRET not in output
+    assert "REDACTED" in output
+
+
+def test_json_excepthook_redacts_traceback_secrets():
+    """Unhandled exception tracebacks in JSON mode must have secrets redacted."""
+    buf = StringIO()
+    h = logging.StreamHandler(buf)
+    h.setFormatter(JsonFormatter())
+    h.addFilter(_secret_filter)
+
+    try:
+        raise RuntimeError(f"Failed to auth with {SECRET}")
+    except RuntimeError:
+        exc_info = sys.exc_info()
+
+    record = logging.LogRecord(
+        name="LiteLLM",
+        level=logging.ERROR,
+        pathname="",
+        lineno=0,
+        msg=str(exc_info[1]),
+        args=(),
+        exc_info=exc_info,
+    )
+    _secret_filter.filter(record)
+    output = h.formatter.format(record)
+    assert SECRET not in output
+    assert "REDACTED" in output