From bc7fef6fdade8a0aa55b32f65c7bebc1b5715705 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 17 Feb 2026 20:31:39 -0800 Subject: [PATCH] Add prompt injection detection policy template + guardrails (#21452) * add SQL injection detection guardrail category * add malicious code injection detection guardrail category * add system prompt extraction detection guardrail category * add jailbreak attempt detection guardrail category * add data exfiltration detection guardrail category * add prompt injection detection policy template --- litellm/policy_templates_backup.json | 121 ++++++++++++++ .../prompt_injection_data_exfiltration.yaml | 123 +++++++++++++++ .../prompt_injection_jailbreak.yaml | 140 +++++++++++++++++ .../prompt_injection_malicious_code.yaml | 147 ++++++++++++++++++ .../categories/prompt_injection_sql.yaml | 120 ++++++++++++++ .../prompt_injection_system_prompt.yaml | 145 +++++++++++++++++ 6 files changed, 796 insertions(+) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_data_exfiltration.yaml create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_jailbreak.yaml create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_malicious_code.yaml create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_sql.yaml create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_system_prompt.yaml diff --git a/litellm/policy_templates_backup.json b/litellm/policy_templates_backup.json index 5b20d86943..9a1eccfa3c 100644 --- a/litellm/policy_templates_backup.json +++ b/litellm/policy_templates_backup.json @@ -831,5 +831,126 @@ ], "guardrails_remove": [] } + }, + { + "id": "prompt-injection-detection", + "title": "Prompt Injection Detection", + "description": "Detects and blocks prompt injection attacks including SQL injection, malicious code injection, system prompt extraction, jailbreak attempts, and data exfiltration. Applies pre-call screening to block attacks before they reach the LLM.", + "region": "Global", + "icon": "ShieldExclamationIcon", + "iconColor": "text-red-500", + "iconBg": "bg-red-50", + "guardrails": [ + "prompt-injection-sql", + "prompt-injection-malicious-code", + "prompt-injection-system-prompt", + "prompt-injection-jailbreak", + "prompt-injection-data-exfiltration" + ], + "complexity": "Medium", + "guardrailDefinitions": [ + { + "guardrail_name": "prompt-injection-sql", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "prompt_injection_sql", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Blocks SQL injection attempts in prompts (DROP TABLE, UNION SELECT, OR 1=1, etc.)" + } + }, + { + "guardrail_name": "prompt-injection-malicious-code", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "prompt_injection_malicious_code", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Blocks malicious code injection attempts (shell commands, reverse shells, script injection, encoded payloads)" + } + }, + { + "guardrail_name": "prompt-injection-system-prompt", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "prompt_injection_system_prompt", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Blocks system prompt extraction and instruction override attempts (ignore previous instructions, reveal your prompt, etc.)" + } + }, + { + "guardrail_name": "prompt-injection-jailbreak", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "prompt_injection_jailbreak", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Blocks jailbreak attempts (DAN mode, developer mode, safety bypass, token smuggling)" + } + }, + { + "guardrail_name": "prompt-injection-data-exfiltration", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "prompt_injection_data_exfiltration", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Blocks data exfiltration attempts (extract training data, dump database, steal credentials, etc.)" + } + } + ], + "templateData": { + "policy_name": "prompt-injection-detection", + "description": "Prompt injection detection policy. Blocks SQL injection, malicious code injection, system prompt extraction, jailbreak attempts, and data exfiltration in prompts before they reach the LLM.", + "guardrails_add": [ + "prompt-injection-sql", + "prompt-injection-malicious-code", + "prompt-injection-system-prompt", + "prompt-injection-jailbreak", + "prompt-injection-data-exfiltration" + ], + "guardrails_remove": [] + } } ] diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_data_exfiltration.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_data_exfiltration.yaml new file mode 100644 index 0000000000..31a2980521 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_data_exfiltration.yaml @@ -0,0 +1,123 @@ +# Data Exfiltration Detection - Detects attempts to extract sensitive data via prompts +# Uses conditional logic: BLOCK if an identifier word + block word appear in same sentence +category_name: "prompt_injection_data_exfiltration" +description: "Detects attempts to extract training data, internal information, or user data through prompts" +default_action: "BLOCK" + +# IDENTIFIER WORDS - Data access and extraction actions +identifier_words: + - "extract" + - "leak" + - "exfiltrate" + - "dump" + - "retrieve" + - "access" + - "steal" + - "obtain" + - "harvest" + - "scrape" + - "collect" + - "gather" + - "expose" + - "export" + - "download" + - "copy" + - "siphon" + +# ADDITIONAL BLOCK WORDS - Sensitive data targets +additional_block_words: + - "training data" + - "training set" + - "training examples" + - "internal data" + - "internal information" + - "database contents" + - "database records" + - "user data" + - "user records" + - "user information" + - "personal data" + - "private information" + - "private data" + - "confidential data" + - "confidential information" + - "sensitive data" + - "sensitive information" + - "customer data" + - "customer records" + - "employee data" + - "employee records" + - "credentials" + - "passwords" + - "API keys" + - "access tokens" + - "secret keys" + - "encryption keys" + - "private keys" + - "session tokens" + - "authentication tokens" + - "model weights" + - "model parameters" + - "source code" + - "proprietary code" + - "intellectual property" + +# ALWAYS BLOCK - Explicit data exfiltration patterns (blocked regardless of context) +always_block_keywords: + - keyword: "extract training data" + severity: "high" + - keyword: "leak internal data" + severity: "high" + - keyword: "dump database" + severity: "high" + - keyword: "dump all records" + severity: "high" + - keyword: "show all user data" + severity: "high" + - keyword: "list all users" + severity: "medium" + - keyword: "list all passwords" + severity: "high" + - keyword: "show all API keys" + severity: "high" + - keyword: "reveal all credentials" + severity: "high" + - keyword: "expose customer data" + severity: "high" + - keyword: "steal user information" + severity: "high" + - keyword: "exfiltrate data" + severity: "high" + - keyword: "harvest email addresses" + severity: "high" + - keyword: "scrape personal information" + severity: "high" + - keyword: "extract model weights" + severity: "high" + - keyword: "steal source code" + severity: "high" + - keyword: "copy proprietary code" + severity: "high" + - keyword: "download private data" + severity: "high" + - keyword: "access confidential files" + severity: "high" + - keyword: "retrieve secret keys" + severity: "high" + +# Exception patterns (legitimate uses) +exceptions: + - "data extraction pipeline" + - "ETL" + - "data science" + - "data engineering" + - "data migration" + - "backup and recovery" + - "data export feature" + - "GDPR data request" + - "right to access" + - "data portability" + - "legitimate data access" + - "authorized data retrieval" + - "data loss prevention" + - "DLP" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_jailbreak.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_jailbreak.yaml new file mode 100644 index 0000000000..f30269571a --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_jailbreak.yaml @@ -0,0 +1,140 @@ +# Jailbreak Attempt Detection - Detects common jailbreak patterns +# Uses conditional logic: BLOCK if an identifier word + block word appear in same sentence +category_name: "prompt_injection_jailbreak" +description: "Detects jailbreak attempts including DAN, roleplay attacks, and safety bypass requests" +default_action: "BLOCK" + +# IDENTIFIER WORDS - Roleplay/persona manipulation actions +identifier_words: + - "pretend" + - "roleplay" + - "act as" + - "imagine you are" + - "suppose you are" + - "you are now" + - "behave as" + - "respond as" + - "simulate" + - "emulate" + - "impersonate" + - "transform into" + - "become" + - "play the role" + - "switch to" + - "enter" + - "enable" + - "activate" + - "turn on" + +# ADDITIONAL BLOCK WORDS - Jailbreak-related terms +additional_block_words: + - "no restrictions" + - "no rules" + - "no guidelines" + - "no filters" + - "no limitations" + - "no boundaries" + - "no censorship" + - "no moderation" + - "no safety" + - "no ethical" + - "bypass" + - "override" + - "unrestricted" + - "unfiltered" + - "uncensored" + - "unmoderated" + - "unlimited" + - "without limits" + - "without restrictions" + - "without rules" + - "without guidelines" + - "without filters" + - "anything mode" + - "god mode" + - "developer mode" + - "debug mode" + - "test mode" + - "admin mode" + - "root mode" + - "superuser" + - "evil mode" + - "chaos mode" + - "unrestricted mode" + - "opposite mode" + +# ALWAYS BLOCK - Explicit jailbreak patterns (blocked regardless of context) +always_block_keywords: + # DAN and known jailbreaks + - keyword: "DAN mode" + severity: "high" + - keyword: "Do Anything Now" + severity: "high" + - keyword: "developer mode enabled" + severity: "high" + - keyword: "jailbreak" + severity: "high" + - keyword: "jailbroken" + severity: "high" + - keyword: "bypass safety" + severity: "high" + - keyword: "bypass filters" + severity: "high" + - keyword: "bypass content filter" + severity: "high" + - keyword: "bypass moderation" + severity: "high" + - keyword: "ignore safety guidelines" + severity: "high" + - keyword: "ignore content policy" + severity: "high" + - keyword: "ignore ethical guidelines" + severity: "high" + - keyword: "without restrictions" + severity: "medium" + - keyword: "without safety" + severity: "high" + - keyword: "without filters" + severity: "medium" + - keyword: "remove all restrictions" + severity: "high" + - keyword: "disable safety" + severity: "high" + - keyword: "disable content filter" + severity: "high" + - keyword: "disable moderation" + severity: "high" + - keyword: "turn off safety" + severity: "high" + - keyword: "turn off filters" + severity: "high" + + # Token smuggling / encoding tricks + - keyword: "respond in base64" + severity: "high" + - keyword: "encode your response" + severity: "medium" + - keyword: "answer in rot13" + severity: "high" + - keyword: "respond in hex" + severity: "high" + - keyword: "use pig latin" + severity: "medium" + - keyword: "respond backwards" + severity: "medium" + - keyword: "spell it out letter by letter" + severity: "medium" + +# Exception patterns (legitimate uses) +exceptions: + - "jailbreak prevention" + - "security research" + - "red team" + - "red teaming" + - "AI safety research" + - "AI alignment" + - "prompt injection defense" + - "how to prevent jailbreaks" + - "jailbreak detection" + - "content moderation research" + - "safety evaluation" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_malicious_code.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_malicious_code.yaml new file mode 100644 index 0000000000..1045208cea --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_malicious_code.yaml @@ -0,0 +1,147 @@ +# Malicious Code Injection Detection - Detects attempts to inject executable code +# Uses conditional logic: BLOCK if an identifier word + block word appear in same sentence +category_name: "prompt_injection_malicious_code" +description: "Detects attempts to inject or execute malicious code via prompts" +default_action: "BLOCK" + +# IDENTIFIER WORDS - Code execution actions +identifier_words: + - "execute" + - "run" + - "eval" + - "exec" + - "spawn" + - "system" + - "shell" + - "invoke" + - "call" + - "launch" + - "process" + - "compile" + - "interpret" + - "inject" + +# ADDITIONAL BLOCK WORDS - Dangerous commands and patterns +additional_block_words: + # Shell commands + - "rm -rf" + - "rm -f" + - "chmod 777" + - "chmod +x" + - "sudo" + - "su root" + - "curl | bash" + - "curl | sh" + - "wget -O -" + - "wget | sh" + - "/etc/passwd" + - "/etc/shadow" + - "mkfs" + - "dd if=" + - "> /dev/sda" + - "fork bomb" + - "crontab" + - "nc -e" + - "netcat" + - "ncat" + - "reverse shell" + + # Script injection + - "powershell -enc" + - "powershell -e" + - "base64 -d" + - "base64 --decode" + - "