From ca229fe030f009b3bcb8aaef3df6b4c2f1c3a85c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 7 Nov 2025 18:14:58 -0800 Subject: [PATCH] [Feat] LiteLLM Guardrail - UI Fix, ensure you can see UI Friendly name for PII Patterns (#16382) * fix safe dumps * add patterns.json * add PrebuiltPattern * add test patterns --- litellm/litellm_core_utils/safe_json_dumps.py | 17 ++ .../litellm_content_filter/patterns.json | 124 ++++++++++++ .../litellm_content_filter/patterns.py | 190 +++++------------- .../test_safe_json_dumps.py | 107 ++++++++++ .../content_filter/test_patterns.py | 157 +++++++++++++++ .../guardrails/add_guardrail_form.tsx | 1 + .../ContentFilterConfiguration.tsx | 5 + .../content_filter/PatternModal.tsx | 11 +- .../content_filter/PatternTable.tsx | 2 + .../guardrails/content_filter/types.ts | 2 + 10 files changed, 472 insertions(+), 144 deletions(-) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index c714e36b5f..bd3bcc9797 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -42,8 +42,25 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: seen.remove(id(obj)) return result else: + # Check if object is a Pydantic model and convert to dict + if hasattr(obj, "model_dump"): + # Pydantic v2 + try: + seen.remove(id(obj)) + return _serialize(obj.model_dump(), seen, depth) + except Exception: + pass + elif hasattr(obj, "dict"): + # Pydantic v1 + try: + seen.remove(id(obj)) + return _serialize(obj.dict(), seen, depth) + except Exception: + pass + # Fall back to string conversion for non-serializable objects. try: + seen.remove(id(obj)) return str(obj) except Exception: return "Unserializable Object" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json new file mode 100644 index 0000000000..06dabced5a --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -0,0 +1,124 @@ +{ + "patterns": [ + { + "name": "us_ssn", + "display_name": "SSN (Social Security Number)", + "pattern": "\\b\\d{3}-\\d{2}-\\d{4}\\b", + "category": "PII Patterns", + "description": "Detects US Social Security Numbers (XXX-XX-XXXX format)" + }, + { + "name": "us_ssn_no_dash", + "display_name": "SSN (No Dashes)", + "pattern": "\\b(?!000|666|9\\d{2})\\d{3}(?!00)\\d{2}(?!0000)\\d{4}\\b", + "category": "PII Patterns", + "description": "Detects US SSN without dashes (XXXXXXXXX format)" + }, + { + "name": "email", + "display_name": "Email Address", + "pattern": "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", + "category": "PII Patterns", + "description": "Detects email addresses" + }, + { + "name": "us_phone", + "display_name": "Phone Number (US)", + "pattern": "\\b(?:\\+?1[\\s.-]?)?\\(?([0-9]{3})\\)?[\\s.-]?([0-9]{3})[\\s.-]?([0-9]{4})\\b", + "category": "PII Patterns", + "description": "Detects US phone numbers in various formats" + }, + { + "name": "visa", + "display_name": "Credit Card (Visa)", + "pattern": "\\b4\\d{3}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b", + "category": "Payment Card Patterns", + "description": "Detects Visa credit card numbers" + }, + { + "name": "mastercard", + "display_name": "Credit Card (Mastercard)", + "pattern": "\\b5[1-5]\\d{2}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b", + "category": "Payment Card Patterns", + "description": "Detects Mastercard credit card numbers" + }, + { + "name": "amex", + "display_name": "Credit Card (American Express)", + "pattern": "\\b3[47]\\d{2}[\\s\\-]?\\d{6}[\\s\\-]?\\d{5}\\b", + "category": "Payment Card Patterns", + "description": "Detects American Express credit card numbers" + }, + { + "name": "discover", + "display_name": "Credit Card (Discover)", + "pattern": "\\b6(?:011|5\\d{2})[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b", + "category": "Payment Card Patterns", + "description": "Detects Discover credit card numbers" + }, + { + "name": "credit_card", + "display_name": "Credit Card (Any)", + "pattern": "(?:\\b4\\d{3}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b|\\b5[1-5]\\d{2}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b|\\b3[47]\\d{2}[\\s\\-]?\\d{6}[\\s\\-]?\\d{5}\\b|\\b6(?:011|5\\d{2})[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}[\\s\\-]?\\d{4}\\b)", + "category": "Payment Card Patterns", + "description": "Detects any major credit card number" + }, + { + "name": "aws_access_key", + "display_name": "AWS Access Key", + "pattern": "\\b(AKIA[0-9A-Z]{16})\\b", + "category": "Credential Patterns", + "description": "Detects AWS access keys (AKIA...)" + }, + { + "name": "aws_secret_key", + "display_name": "AWS Secret Key", + "pattern": "\\b([A-Za-z0-9/+=]{40})\\b", + "category": "Credential Patterns", + "description": "Detects AWS secret keys (40 characters)" + }, + { + "name": "github_token", + "display_name": "GitHub Token", + "pattern": "\\b(gh[ps]_[a-zA-Z0-9]{36})\\b", + "category": "Credential Patterns", + "description": "Detects GitHub personal access tokens" + }, + { + "name": "slack_token", + "display_name": "Slack Token", + "pattern": "\\b(xox[pboa]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,32})\\b", + "category": "Credential Patterns", + "description": "Detects Slack API tokens" + }, + { + "name": "generic_api_key", + "display_name": "API Key (Generic)", + "pattern": "\\b([Aa][Pp][Ii][-_]?[Kk][Ee][Yy][\\s:=]+['\"]?[A-Za-z0-9_\\-]{20,}['\"]?)\\b", + "category": "Credential Patterns", + "description": "Detects generic API key patterns" + }, + { + "name": "ipv4", + "display_name": "IP Address (IPv4)", + "pattern": "\\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\b", + "category": "Network Patterns", + "description": "Detects IPv4 addresses" + }, + { + "name": "ipv6", + "display_name": "IP Address (IPv6)", + "pattern": "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b", + "category": "Network Patterns", + "description": "Detects IPv6 addresses" + }, + { + "name": "url", + "display_name": "URL", + "pattern": "\\b(?:https?://|www\\.)[^\\s/$.?#].[^\\s]*\\b", + "category": "Network Patterns", + "description": "Detects URLs (http/https)" + } + ] +} + diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py index cfdb269411..18cca3a0ce 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py @@ -1,102 +1,42 @@ """ Prebuilt regex patterns for content filtering. -This module contains predefined regex patterns for detecting sensitive information -like SSNs, credit cards, API keys, etc. +This module loads predefined regex patterns from patterns.json for detecting +sensitive information like SSNs, credit cards, API keys, etc. """ +import json +import os import re from enum import Enum -from typing import Dict, List, Pattern +from typing import Dict, List, Optional, Pattern -# US Social Security Number patterns -US_SSN_PATTERN = r"\b\d{3}-\d{2}-\d{4}\b" # Format: 123-45-6789 -US_SSN_NO_DASH_PATTERN = r"\b(?!000|666|9\d{2})\d{3}(?!00)\d{2}(?!0000)\d{4}\b" # Format: 123456789 (with validation) -# Credit Card patterns -VISA_PATTERN = r"\b4\d{3}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b" # Visa starts with 4 -MASTERCARD_PATTERN = r"\b5[1-5]\d{2}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b" # Mastercard starts with 51-55 -AMEX_PATTERN = r"\b3[47]\d{2}[\s\-]?\d{6}[\s\-]?\d{5}\b" # Amex starts with 34 or 37 -DISCOVER_PATTERN = r"\b6(?:011|5\d{2})[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b" # Discover starts with 6011 or 65 +def _load_patterns_from_json() -> Dict: + """Load pattern definitions from patterns.json file""" + json_path = os.path.join(os.path.dirname(__file__), "patterns.json") + with open(json_path, "r") as f: + return json.load(f) -# Email pattern -EMAIL_PATTERN = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b" -# Phone number patterns (US) -US_PHONE_PATTERN = r"\b(?:\+?1[\s.-]?)?\(?([0-9]{3})\)?[\s.-]?([0-9]{3})[\s.-]?([0-9]{4})\b" - -# API Key patterns (common formats) -AWS_ACCESS_KEY_PATTERN = r"\b(AKIA[0-9A-Z]{16})\b" # AWS Access Key ID -AWS_SECRET_KEY_PATTERN = r"\b([A-Za-z0-9/+=]{40})\b" # AWS Secret Access Key (generic 40 char) -GITHUB_TOKEN_PATTERN = r"\b(gh[ps]_[a-zA-Z0-9]{36})\b" # GitHub Personal Access Token -SLACK_TOKEN_PATTERN = r"\b(xox[pboa]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,32})\b" # Slack tokens -GENERIC_API_KEY_PATTERN = r"\b([Aa][Pp][Ii][-_]?[Kk][Ee][Yy][\s:=]+['\"]?[A-Za-z0-9_\-]{20,}['\"]?)\b" - -# IP Address patterns -IPV4_PATTERN = r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b" -IPV6_PATTERN = r"\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b" - -# URL patterns -URL_PATTERN = r"\b(?:https?://|www\.)[^\s/$.?#].[^\s]*\b" +# Load patterns from JSON +_PATTERNS_DATA = _load_patterns_from_json() class PrebuiltPatternName(str, Enum): - """Enum for prebuilt pattern names""" - # SSN patterns - US_SSN = "us_ssn" - US_SSN_NO_DASH = "us_ssn_no_dash" - - # Credit card patterns - VISA = "visa" - MASTERCARD = "mastercard" - AMEX = "amex" - DISCOVER = "discover" - CREDIT_CARD = "credit_card" - - # Contact information - EMAIL = "email" - US_PHONE = "us_phone" - - # API keys and secrets - AWS_ACCESS_KEY = "aws_access_key" - AWS_SECRET_KEY = "aws_secret_key" - GITHUB_TOKEN = "github_token" - SLACK_TOKEN = "slack_token" - GENERIC_API_KEY = "generic_api_key" - - # Network identifiers - IPV4 = "ipv4" - IPV6 = "ipv6" - URL = "url" + """Enum for prebuilt pattern names - dynamically generated from JSON""" + pass +# Dynamically create enum values from JSON +for pattern_data in _PATTERNS_DATA["patterns"]: + setattr(PrebuiltPatternName, pattern_data["name"].upper(), pattern_data["name"]) + + +# Build lookup dictionaries from JSON PREBUILT_PATTERNS: Dict[str, str] = { - # SSN patterns - PrebuiltPatternName.US_SSN.value: US_SSN_PATTERN, - PrebuiltPatternName.US_SSN_NO_DASH.value: US_SSN_NO_DASH_PATTERN, - - # Credit card patterns - PrebuiltPatternName.VISA.value: VISA_PATTERN, - PrebuiltPatternName.MASTERCARD.value: MASTERCARD_PATTERN, - PrebuiltPatternName.AMEX.value: AMEX_PATTERN, - PrebuiltPatternName.DISCOVER.value: DISCOVER_PATTERN, - PrebuiltPatternName.CREDIT_CARD.value: rf"(?:{VISA_PATTERN}|{MASTERCARD_PATTERN}|{AMEX_PATTERN}|{DISCOVER_PATTERN})", - - # Contact information - PrebuiltPatternName.EMAIL.value: EMAIL_PATTERN, - PrebuiltPatternName.US_PHONE.value: US_PHONE_PATTERN, - - # API keys and secrets - PrebuiltPatternName.AWS_ACCESS_KEY.value: AWS_ACCESS_KEY_PATTERN, - PrebuiltPatternName.AWS_SECRET_KEY.value: AWS_SECRET_KEY_PATTERN, - PrebuiltPatternName.GITHUB_TOKEN.value: GITHUB_TOKEN_PATTERN, - PrebuiltPatternName.SLACK_TOKEN.value: SLACK_TOKEN_PATTERN, - PrebuiltPatternName.GENERIC_API_KEY.value: GENERIC_API_KEY_PATTERN, - - # Network identifiers - PrebuiltPatternName.IPV4.value: IPV4_PATTERN, - PrebuiltPatternName.IPV6.value: IPV6_PATTERN, - PrebuiltPatternName.URL.value: URL_PATTERN, + pattern_data["name"]: pattern_data["pattern"] + for pattern_data in _PATTERNS_DATA["patterns"] } @@ -123,7 +63,7 @@ def get_compiled_pattern(pattern_name: str) -> Pattern: return re.compile(PREBUILT_PATTERNS[pattern_name], re.IGNORECASE) -def get_all_pattern_names(): +def get_all_pattern_names() -> List[str]: """ Get a list of all available prebuilt pattern names. @@ -133,54 +73,26 @@ def get_all_pattern_names(): return list(PREBUILT_PATTERNS.keys()) -# Pattern categories for UI organization -PATTERN_CATEGORIES: Dict[str, List[str]] = { - "PII Patterns": [ - PrebuiltPatternName.US_SSN.value, - PrebuiltPatternName.EMAIL.value, - PrebuiltPatternName.US_PHONE.value, - ], - "Payment Card Patterns": [ - PrebuiltPatternName.VISA.value, - PrebuiltPatternName.MASTERCARD.value, - PrebuiltPatternName.AMEX.value, - PrebuiltPatternName.DISCOVER.value, - PrebuiltPatternName.CREDIT_CARD.value, - ], - "Credential Patterns": [ - PrebuiltPatternName.AWS_ACCESS_KEY.value, - PrebuiltPatternName.AWS_SECRET_KEY.value, - PrebuiltPatternName.GITHUB_TOKEN.value, - PrebuiltPatternName.SLACK_TOKEN.value, - PrebuiltPatternName.GENERIC_API_KEY.value, - ], - "Network Patterns": [ - PrebuiltPatternName.IPV4.value, - PrebuiltPatternName.IPV6.value, - PrebuiltPatternName.URL.value, - ], +# Build category mapping from JSON +PATTERN_CATEGORIES: Dict[str, List[str]] = {} +for pattern_data in _PATTERNS_DATA["patterns"]: + category = pattern_data["category"] + if category not in PATTERN_CATEGORIES: + PATTERN_CATEGORIES[category] = [] + PATTERN_CATEGORIES[category].append(pattern_data["name"]) + + +# Build display names mapping from JSON +PATTERN_DISPLAY_NAMES: Dict[str, str] = { + pattern_data["name"]: pattern_data["display_name"] + for pattern_data in _PATTERNS_DATA["patterns"] } -# Pattern descriptions for UI display +# Build descriptions mapping from JSON PATTERN_DESCRIPTIONS: Dict[str, str] = { - PrebuiltPatternName.US_SSN.value: "Detects US Social Security Numbers (XXX-XX-XXXX format)", - PrebuiltPatternName.US_SSN_NO_DASH.value: "Detects US SSN without dashes (XXXXXXXXX format)", - PrebuiltPatternName.EMAIL.value: "Detects email addresses", - PrebuiltPatternName.US_PHONE.value: "Detects US phone numbers in various formats", - PrebuiltPatternName.VISA.value: "Detects Visa credit card numbers", - PrebuiltPatternName.MASTERCARD.value: "Detects Mastercard credit card numbers", - PrebuiltPatternName.AMEX.value: "Detects American Express credit card numbers", - PrebuiltPatternName.DISCOVER.value: "Detects Discover credit card numbers", - PrebuiltPatternName.CREDIT_CARD.value: "Detects any major credit card number", - PrebuiltPatternName.AWS_ACCESS_KEY.value: "Detects AWS access keys (AKIA...)", - PrebuiltPatternName.AWS_SECRET_KEY.value: "Detects AWS secret keys (40 characters)", - PrebuiltPatternName.GITHUB_TOKEN.value: "Detects GitHub personal access tokens", - PrebuiltPatternName.SLACK_TOKEN.value: "Detects Slack API tokens", - PrebuiltPatternName.GENERIC_API_KEY.value: "Detects generic API key patterns", - PrebuiltPatternName.IPV4.value: "Detects IPv4 addresses", - PrebuiltPatternName.IPV6.value: "Detects IPv6 addresses", - PrebuiltPatternName.URL.value: "Detects URLs (http/https)", + pattern_data["name"]: pattern_data["description"] + for pattern_data in _PATTERNS_DATA["patterns"] } @@ -189,19 +101,15 @@ def get_pattern_metadata() -> List[Dict[str, str]]: Return pattern metadata for UI display. Returns: - List of dictionaries containing pattern name, category, and description + List of dictionaries containing pattern name, display_name, category, and description """ - prebuilt_patterns = [] - for category, pattern_names in PATTERN_CATEGORIES.items(): - for pattern_name in pattern_names: - if pattern_name in PREBUILT_PATTERNS: - prebuilt_patterns.append({ - "name": pattern_name, - "category": category, - "description": PATTERN_DESCRIPTIONS.get( - pattern_name, - f"Detects {pattern_name.replace('_', ' ').title()}" - ), - }) - return prebuilt_patterns + return [ + { + "name": pattern_data["name"], + "display_name": pattern_data["display_name"], + "category": pattern_data["category"], + "description": pattern_data["description"], + } + for pattern_data in _PATTERNS_DATA["patterns"] + ] diff --git a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py index 56937e6266..d28d63ba50 100644 --- a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py +++ b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py @@ -138,3 +138,110 @@ def test_non_standard_dict_keys_complex(): traceback.print_exc() raise e + + +def test_pydantic_model_serialization(): + """ + Test that safe_dumps properly serializes Pydantic models to dictionaries + instead of converting them to string representations. + This prevents the validation error: "Input should be a valid dictionary or instance of ContentFilterPattern" + """ + from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern + + pattern = ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + name="email", + action=ContentFilterAction.BLOCK + ) + + result = safe_dumps({"patterns": [pattern]}) + + parsed = json.loads(result) + assert "patterns" in parsed + assert isinstance(parsed["patterns"], list) + assert len(parsed["patterns"]) == 1 + assert isinstance(parsed["patterns"][0], dict) + assert parsed["patterns"][0]["pattern_type"] == "prebuilt" + assert parsed["patterns"][0]["pattern_name"] == "email" + assert parsed["patterns"][0]["name"] == "email" + assert parsed["patterns"][0]["action"] == "BLOCK" + + assert "pattern_type='prebuilt'" not in result + + +def test_list_of_pydantic_models(): + """ + Test that safe_dumps properly serializes a list of Pydantic models. + """ + from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern + + patterns = [ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + name="email", + action=ContentFilterAction.BLOCK + ), + ContentFilterPattern( + pattern_type="regex", + pattern=r"\d{3}-\d{2}-\d{4}", + name="ssn", + action=ContentFilterAction.MASK + ), + ] + + result = safe_dumps({"patterns": patterns}) + + parsed = json.loads(result) + assert "patterns" in parsed + assert isinstance(parsed["patterns"], list) + assert len(parsed["patterns"]) == 2 + + assert parsed["patterns"][0]["pattern_type"] == "prebuilt" + assert parsed["patterns"][0]["pattern_name"] == "email" + assert parsed["patterns"][0]["action"] == "BLOCK" + + assert parsed["patterns"][1]["pattern_type"] == "regex" + assert parsed["patterns"][1]["pattern"] == r"\d{3}-\d{2}-\d{4}" + assert parsed["patterns"][1]["action"] == "MASK" + + assert "ContentFilterPattern" not in result + + +def test_nested_pydantic_models_in_dict(): + """ + Test that safe_dumps properly serializes nested dictionaries containing Pydantic models. + This simulates the guardrail litellm_params structure. + """ + from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern + + litellm_params = { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + name="email", + action=ContentFilterAction.BLOCK + ) + ], + "metadata": { + "nested": True, + "count": 1 + } + } + + result = safe_dumps(litellm_params) + + parsed = json.loads(result) + assert parsed["guardrail"] == "litellm_content_filter" + assert parsed["mode"] == "pre_call" + assert isinstance(parsed["patterns"], list) + assert len(parsed["patterns"]) == 1 + assert isinstance(parsed["patterns"][0], dict) + assert parsed["patterns"][0]["pattern_type"] == "prebuilt" + + assert "ContentFilterPattern" not in result + assert "pattern_type='prebuilt'" not in result diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py new file mode 100644 index 0000000000..3380cefa65 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py @@ -0,0 +1,157 @@ +""" +Tests for content filter pattern loading from JSON +""" +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../") +) + +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( + PATTERN_CATEGORIES, + PATTERN_DESCRIPTIONS, + PATTERN_DISPLAY_NAMES, + PREBUILT_PATTERNS, + get_all_pattern_names, + get_compiled_pattern, + get_pattern_metadata, +) + + +def test_patterns_loaded_from_json(): + """ + Test that patterns are successfully loaded from JSON file + """ + assert len(PREBUILT_PATTERNS) > 0 + assert len(PATTERN_CATEGORIES) > 0 + assert len(PATTERN_DISPLAY_NAMES) > 0 + assert len(PATTERN_DESCRIPTIONS) > 0 + + +def test_pattern_metadata_structure(): + """ + Test that get_pattern_metadata returns correctly structured data + """ + patterns = get_pattern_metadata() + + assert len(patterns) > 0 + + for pattern in patterns: + assert "name" in pattern + assert "display_name" in pattern + assert "category" in pattern + assert "description" in pattern + + assert isinstance(pattern["name"], str) + assert isinstance(pattern["display_name"], str) + assert isinstance(pattern["category"], str) + assert isinstance(pattern["description"], str) + + +def test_display_names_user_friendly(): + """ + Test that display names are user-friendly and different from internal names + """ + patterns = get_pattern_metadata() + + ssn_pattern = next((p for p in patterns if p["name"] == "us_ssn"), None) + assert ssn_pattern is not None + assert ssn_pattern["display_name"] == "SSN (Social Security Number)" + + email_pattern = next((p for p in patterns if p["name"] == "email"), None) + assert email_pattern is not None + assert email_pattern["display_name"] == "Email Address" + + +def test_pattern_compilation(): + """ + Test that patterns can be compiled into regex objects + """ + pattern_names = get_all_pattern_names() + + assert len(pattern_names) > 0 + + for pattern_name in pattern_names: + compiled_pattern = get_compiled_pattern(pattern_name) + assert compiled_pattern is not None + assert hasattr(compiled_pattern, "match") + + +def test_pattern_compilation_error(): + """ + Test that invalid pattern names raise ValueError + """ + with pytest.raises(ValueError, match="Unknown pattern name"): + get_compiled_pattern("invalid_pattern_name") + + +def test_categories_contain_patterns(): + """ + Test that each category contains valid pattern names + """ + all_pattern_names = set(PREBUILT_PATTERNS.keys()) + + for category, patterns in PATTERN_CATEGORIES.items(): + assert len(patterns) > 0 + for pattern_name in patterns: + assert pattern_name in all_pattern_names + + +def test_json_file_exists(): + """ + Test that patterns.json file exists and is valid JSON + """ + json_path = os.path.join( + os.path.dirname(__file__), + "../../../../../../litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json" + ) + + assert os.path.exists(json_path) + + with open(json_path, "r") as f: + data = json.load(f) + assert "patterns" in data + assert len(data["patterns"]) > 0 + + +def test_json_pattern_structure(): + """ + Test that each pattern in JSON has required fields + """ + json_path = os.path.join( + os.path.dirname(__file__), + "../../../../../../litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json" + ) + + with open(json_path, "r") as f: + data = json.load(f) + + for pattern in data["patterns"]: + assert "name" in pattern + assert "display_name" in pattern + assert "pattern" in pattern + assert "category" in pattern + assert "description" in pattern + + assert isinstance(pattern["name"], str) + assert isinstance(pattern["display_name"], str) + assert isinstance(pattern["pattern"], str) + assert isinstance(pattern["category"], str) + assert isinstance(pattern["description"], str) + + +def test_all_dictionaries_consistent(): + """ + Test that all pattern dictionaries have consistent keys + """ + pattern_names_from_patterns = set(PREBUILT_PATTERNS.keys()) + pattern_names_from_display = set(PATTERN_DISPLAY_NAMES.keys()) + pattern_names_from_descriptions = set(PATTERN_DESCRIPTIONS.keys()) + + assert pattern_names_from_patterns == pattern_names_from_display + assert pattern_names_from_patterns == pattern_names_from_descriptions + diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index 95aa398f43..55fe800a43 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -49,6 +49,7 @@ interface GuardrailSettings { content_filter_settings?: { prebuilt_patterns: Array<{ name: string; + display_name: string; category: string; description: string; }>; diff --git a/ui/litellm-dashboard/src/components/guardrails/content_filter/ContentFilterConfiguration.tsx b/ui/litellm-dashboard/src/components/guardrails/content_filter/ContentFilterConfiguration.tsx index 9e5a2ffe35..af9723564e 100644 --- a/ui/litellm-dashboard/src/components/guardrails/content_filter/ContentFilterConfiguration.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/content_filter/ContentFilterConfiguration.tsx @@ -14,6 +14,7 @@ const { Title, Text } = Typography; interface PrebuiltPattern { name: string; + display_name: string; category: string; description: string; } @@ -22,6 +23,7 @@ interface Pattern { id: string; type: "prebuilt" | "custom"; name: string; + display_name?: string; pattern?: string; action: "BLOCK" | "MASK"; } @@ -84,10 +86,13 @@ const ContentFilterConfiguration: React.FC = ({ return; } + const selectedPattern = prebuiltPatterns.find((p) => p.name === selectedPatternName); + onPatternAdd({ id: `pattern-${Date.now()}`, type: "prebuilt", name: selectedPatternName, + display_name: selectedPattern?.display_name, action: patternAction, }); diff --git a/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternModal.tsx b/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternModal.tsx index 8760dfaf92..4f4f2648b5 100644 --- a/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternModal.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternModal.tsx @@ -7,6 +7,7 @@ const { Option } = Select; interface PrebuiltPattern { name: string; + display_name: string; category: string; description: string; } @@ -52,8 +53,12 @@ const PatternModal: React.FC = ({ style={{ width: "100%", marginTop: 8 }} showSearch filterOption={(input, option) => { - if (typeof option?.value === 'string') { - return option.value.toLowerCase().includes(input.toLowerCase()); + const pattern = prebuiltPatterns.find((p) => p.name === option?.value); + if (pattern) { + return ( + pattern.display_name.toLowerCase().includes(input.toLowerCase()) || + pattern.name.toLowerCase().includes(input.toLowerCase()) + ); } return false; }} @@ -66,7 +71,7 @@ const PatternModal: React.FC = ({ {categoryPatterns.map((pattern) => ( ))} diff --git a/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternTable.tsx b/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternTable.tsx index 6e969649cf..a3c0580133 100644 --- a/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternTable.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/content_filter/PatternTable.tsx @@ -10,6 +10,7 @@ interface Pattern { id: string; type: "prebuilt" | "custom"; name: string; + display_name?: string; pattern?: string; action: "BLOCK" | "MASK"; } @@ -41,6 +42,7 @@ const PatternTable: React.FC = ({ title: "Pattern name", dataIndex: "name", key: "name", + render: (_: string, record: Pattern) => record.display_name || record.name, }, { title: "Regex pattern", diff --git a/ui/litellm-dashboard/src/components/guardrails/content_filter/types.ts b/ui/litellm-dashboard/src/components/guardrails/content_filter/types.ts index 8513143fde..26e4478dc1 100644 --- a/ui/litellm-dashboard/src/components/guardrails/content_filter/types.ts +++ b/ui/litellm-dashboard/src/components/guardrails/content_filter/types.ts @@ -4,6 +4,7 @@ export interface PrebuiltPattern { name: string; + display_name: string; category: string; description: string; } @@ -12,6 +13,7 @@ export interface Pattern { id: string; type: "prebuilt" | "custom"; name: string; + display_name?: string; pattern?: string; action: "BLOCK" | "MASK"; }