Merge pull request #26957 from stuxf/chore/guardrail-coverage

chore(guardrails): cover multimodal + Responses-API content shapes
This commit is contained in:
yuneng-jiang
2026-05-04 18:01:27 -07:00
committed by GitHub
20 changed files with 1903 additions and 300 deletions
+17 -9
View File
@@ -11,6 +11,10 @@ from typing import Literal
import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
is_text_content_call_type,
iter_message_text,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from fastapi import HTTPException
@@ -73,10 +77,9 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
- check if user id part of blocked list
"""
self.print_verbose("Inside Banned Keyword List Pre-Call Hook")
if call_type == "completion" and "messages" in data:
for m in data["messages"]:
if "content" in m and isinstance(m["content"], str):
self.test_violation(test_str=m["content"])
if is_text_content_call_type(call_type):
for text in iter_message_text(data):
self.test_violation(test_str=text)
except HTTPException as e:
raise e
@@ -93,11 +96,16 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
response,
):
if isinstance(response, litellm.ModelResponse) and isinstance(
response.choices[0], litellm.utils.Choices
):
for word in self.banned_keywords_list:
self.test_violation(test_str=response.choices[0].message.content or "")
if not isinstance(response, litellm.ModelResponse):
return
for choice in response.choices:
if not isinstance(choice, litellm.utils.Choices):
continue
message = getattr(choice, "message", None)
content = getattr(message, "content", None)
if isinstance(content, str):
self.test_violation(test_str=content)
async def async_post_call_streaming_hook(
self,
@@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@@ -94,11 +95,9 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if "messages" in data and isinstance(data["messages"], list):
text = ""
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:
document = self.language_document(content=text, type_=self.document_type)
request = self.moderate_text_request(
@@ -19,6 +19,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@@ -37,11 +38,8 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
text = ""
if "messages" in data and isinstance(data["messages"], list):
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
from litellm.proxy.proxy_server import llm_router
@@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import walk_user_text
GUARDRAIL_NAME = "hide_secrets"
@@ -473,23 +474,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
if await self.should_run_check(user_api_key_dict) is False:
return
if "messages" in data and isinstance(data["messages"], list):
for message in data["messages"]:
if "content" in message and isinstance(message["content"], str):
detected_secrets = self.scan_message_for_secrets(message["content"])
# Covers multimodal list content + Responses-API input.
def _redact_message_text(text: str) -> str:
detected_secrets = self.scan_message_for_secrets(text)
for secret in detected_secrets:
text = text.replace(secret["value"], "[REDACTED]")
if detected_secrets:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
return text
for secret in detected_secrets:
message["content"] = message["content"].replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
else:
verbose_proxy_logger.debug("No secrets detected on input.")
walk_user_text(data, _redact_message_text)
if "prompt" in data:
if isinstance(data["prompt"], str):
@@ -504,11 +501,15 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
elif isinstance(data["prompt"], list):
for item in data["prompt"]:
# Index back into the list — assigning to ``item`` would only
# rebind the loop variable and leave ``data["prompt"]``
# carrying the unredacted secret.
for idx, item in enumerate(data["prompt"]):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
item = item.replace(secret["value"], "[REDACTED]")
data["prompt"][idx] = item
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
@@ -517,31 +518,6 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
if "input" in data:
if isinstance(data["input"], str):
detected_secrets = self.scan_message_for_secrets(data["input"])
for secret in detected_secrets:
data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
elif isinstance(data["input"], list):
_input_in_request = data["input"]
for idx, item in enumerate(_input_in_request):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
_input_in_request[idx] = item.replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
verbose_proxy_logger.debug("Data after redacting input %s", data)
# ``data["input"]`` (Responses API and embeddings/moderation) is
# already covered by ``walk_user_text`` above.
return
+236
View File
@@ -0,0 +1,236 @@
"""
Shared helpers for guardrail hooks: extract text from a request body
regardless of whether it uses Chat Completions ``messages``, Responses-API
``input``, or multimodal list-format ``content`` parts.
Hooks that only check ``data["messages"]`` for string content silently
skip the other shapes — these helpers normalise that so every hook sees
every text fragment.
"""
from typing import Any, Callable, Dict, FrozenSet, Iterator, List
# Call types whose body carries free-form chat / prompt text that
# text-content guardrails (banned keywords, content moderation, secret
# detection, …) should inspect. The proxy ingress passes ``route_type``
# straight through as ``call_type``, so the literal values here are
# what the guardrail dispatcher actually receives:
#
# /v1/chat/completions -> "acompletion"
# /v1/responses -> "aresponses"
#
# ``"completion"`` is included for SDK / internal callers that invoke
# ``pre_call_hook`` directly with the sync name. Embedding, moderation,
# audio, and transcription endpoints are deliberately excluded — text
# guardrails on those paths are a separate scope.
TEXT_CONTENT_CALL_TYPES: FrozenSet[str] = frozenset(
{"completion", "acompletion", "aresponses"}
)
def is_text_content_call_type(call_type: str) -> bool:
"""Return True if ``call_type`` carries free-form text that text
guardrails should inspect (Chat Completions or Responses API)."""
return call_type in TEXT_CONTENT_CALL_TYPES
def _iter_text_parts_in_content(content: Any) -> Iterator[str]:
"""Yield text fragments from a ``message.content`` value (string or
multimodal list). Non-text parts (images, audio, …) are skipped."""
if isinstance(content, str):
if content:
yield content
elif isinstance(content, list):
for part in content:
if isinstance(part, str):
# A bare string in a content/input list is itself a text
# fragment (Responses-API mixed-list shape).
if part:
yield part
continue
if not isinstance(part, dict):
continue
if part.get("type") == "text":
text = part.get("text")
if isinstance(text, str) and text:
yield text
def _coerce_input_to_messages(input_value: Any) -> List[Dict[str, Any]]:
"""Coerce a Responses-API ``data["input"]`` value into chat-style messages."""
if isinstance(input_value, str):
return [{"role": "user", "content": input_value}]
if isinstance(input_value, list):
if input_value and all(
isinstance(item, dict) and "role" in item for item in input_value
):
return list(input_value)
# Mixed lists (content-part dicts + bare strings) and pure
# string/dict lists all become a single user message; the content
# iterator below handles each element type uniformly.
return [{"role": "user", "content": input_value}]
return []
def _iter_inspection_messages(data: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
"""Yield every message-like dict, walking ``messages`` AND ``input``."""
messages = data.get("messages")
if isinstance(messages, list):
yield from messages
yield from _coerce_input_to_messages(data.get("input"))
def iter_message_text(data: Dict[str, Any]) -> Iterator[str]:
"""Yield every text fragment from ``messages`` AND ``input``.
Walks every role (user, assistant, system, …) — guardrails inspect
the entire conversation, not just user turns.
"""
for message in _iter_inspection_messages(data):
if not isinstance(message, dict):
continue
yield from _iter_text_parts_in_content(message.get("content"))
def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int:
"""Rewrite every text fragment in place via ``visit``.
Mutates ``data["messages"]`` and ``data["input"]``. Returns the number
of fragments visited so callers can short-circuit when nothing was
inspected.
"""
visited = 0
def _rewrite_content(content: Any) -> Any:
nonlocal visited
if isinstance(content, str):
if content:
visited += 1
return visit(content)
return content
if isinstance(content, list):
new_parts: List[Any] = []
for part in content:
if isinstance(part, str) and part:
visited += 1
new_parts.append(visit(part))
elif (
isinstance(part, dict)
and part.get("type") == "text"
and isinstance(part.get("text"), str)
and part["text"]
):
visited += 1
new_parts.append({**part, "text": visit(part["text"])})
else:
new_parts.append(part)
return new_parts
return content
messages = data.get("messages")
if isinstance(messages, list):
for message in messages:
if isinstance(message, dict) and "content" in message:
message["content"] = _rewrite_content(message["content"])
input_value = data.get("input")
if isinstance(input_value, str):
if input_value:
visited += 1
data["input"] = visit(input_value)
return visited
if isinstance(input_value, list):
# List of full messages: rewrite each message's content.
if input_value and all(
isinstance(item, dict) and "role" in item for item in input_value
):
for item in input_value:
if "content" in item:
item["content"] = _rewrite_content(item["content"])
return visited
# List of content parts and/or bare strings: rewrite in place.
for idx, item in enumerate(input_value):
if isinstance(item, str) and item:
visited += 1
input_value[idx] = visit(item)
elif (
isinstance(item, dict)
and item.get("type") == "text"
and isinstance(item.get("text"), str)
and item["text"]
):
visited += 1
input_value[idx] = {**item, "text": visit(item["text"])}
return visited
return visited
def apply_redacted_messages_back(
data: Dict[str, Any], redacted_messages: List[Dict[str, Any]]
) -> None:
"""Write redacted messages back to whichever field(s) the caller used.
Mask/anonymize paths take a synthesised messages list (from
:func:`build_inspection_messages`), get a redacted version back from a
third-party guardrail, and need to rewrite the request body. Writing
only to ``data["messages"]`` leaves the Responses-API ``data["input"]``
field untouched, so the unredacted text still reaches the LLM.
This helper updates both fields when both are present.
"""
if "messages" in data:
data["messages"] = redacted_messages
if isinstance(data.get("input"), str):
text_parts: List[str] = []
for msg in redacted_messages:
if not isinstance(msg, dict):
continue
text_parts.extend(_iter_text_parts_in_content(msg.get("content")))
data["input"] = "\n".join(text_parts)
def has_non_string_content(data: Dict[str, Any]) -> bool:
"""Return True if any inspected content is not a plain string.
Used by hooks whose mask/redact path operates on string offsets and
therefore cannot preserve multimodal non-text parts. Such hooks should
degrade to block-on-detect when this returns True so image/audio parts
are not silently stripped during in-place masking.
"""
messages = data.get("messages")
if isinstance(messages, list):
for message in messages:
if isinstance(message, dict) and not isinstance(
message.get("content"), str
):
if message.get("content") is not None:
return True
input_value = data.get("input")
if input_value is not None and not isinstance(input_value, str):
return True
return False
def build_inspection_messages(data: Dict[str, Any]) -> List[Dict[str, str]]:
"""Synthesize a chat-style messages list for posting to a guardrail API.
Each returned message has a plain-string ``content`` — multimodal text
parts are joined with newlines and Responses-API ``input`` is lifted
into synthetic messages. Messages with no inspectable text are dropped.
Hooks that POST ``{"messages": [...]}`` to an external service should
call this instead of ``data.get("messages", [])`` so the Responses API
and multimodal content are covered.
"""
flattened: List[Dict[str, str]] = []
for message in _iter_inspection_messages(data):
if not isinstance(message, dict):
continue
text = "\n".join(_iter_text_parts_in_content(message.get("content")))
if not text:
continue
role = message.get("role", "user") or "user"
flattened.append({"role": role, "content": text})
return flattened
@@ -22,6 +22,11 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
apply_redacted_messages_back,
build_inspection_messages,
has_non_string_content,
)
from litellm.types.utils import (
CallTypesLiteral,
Choices,
@@ -101,10 +106,11 @@ class AimGuardrail(CustomGuardrail):
user_email=user_email,
litellm_call_id=call_id,
)
# Covers multimodal list content + Responses-API input.
response = await self.async_handler.post(
f"{self.api_base}/fw/v1/analyze",
headers=headers,
json={"messages": data.get("messages", [])},
json={"messages": build_inspection_messages(data)},
)
response.raise_for_status()
res = response.json()
@@ -137,13 +143,31 @@ class AimGuardrail(CustomGuardrail):
redacted_chat = res.get("redacted_chat")
if not redacted_chat:
return data
data["messages"] = [
# Aim returns text-only redacted messages. Overwriting
# ``data["messages"]`` with that would silently strip image/audio
# parts from a multimodal request — degrade to block so the
# multimodal payload is never silently rewritten.
if has_non_string_content(data):
raise HTTPException(
status_code=400,
detail=(
"Aim: anonymize action requested for multimodal input "
"but mask-in-place would drop non-text parts. Send the "
"request with plain string content to use anonymize, "
"or rely on block-mode policies."
),
)
redacted_messages = [
{
"role": message["role"],
"content": message["content"],
}
for message in redacted_chat["all_redacted_messages"]
]
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages`` would let
# unredacted text reach the LLM for ``/v1/responses`` calls.
apply_redacted_messages_back(data, redacted_messages)
return data
async def call_aim_guardrail_on_output(
@@ -162,7 +186,7 @@ class AimGuardrail(CustomGuardrail):
litellm_call_id=call_id,
),
json={
"messages": request_data.get("messages", [])
"messages": build_inspection_messages(request_data)
+ [{"role": "assistant", "content": output}]
},
)
@@ -233,15 +257,33 @@ class AimGuardrail(CustomGuardrail):
user_api_key_dict: UserAPIKeyAuth,
response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse],
) -> Any:
if (
isinstance(response, ModelResponse)
and response.choices
and isinstance(response.choices[0], Choices)
):
content = response.choices[0].message.content or ""
aim_output_guardrail_result = await self.call_aim_guardrail_on_output(
data, content, hook="output", key_alias=user_api_key_dict.key_alias
)
if not (isinstance(response, ModelResponse) and response.choices):
return response
# Inspect every choice — when ``n>1`` the additional completions
# used to bypass Aim entirely because the hook only inspected
# ``choices[0]``. Run inspections concurrently so multi-completion
# responses don't pay an n× latency penalty.
choices_to_inspect = [c for c in response.choices if isinstance(c, Choices)]
if not choices_to_inspect:
return response
# ``return_exceptions=True`` lets every inspection finish even if
# one fails — without it, the first exception would propagate and
# leave the remaining tasks running in the background.
results = await asyncio.gather(
*(
self.call_aim_guardrail_on_output(
data,
choice.message.content or "",
hook="output",
key_alias=user_api_key_dict.key_alias,
)
for choice in choices_to_inspect
),
return_exceptions=True,
)
for choice, aim_output_guardrail_result in zip(choices_to_inspect, results):
if isinstance(aim_output_guardrail_result, BaseException):
raise aim_output_guardrail_result
if aim_output_guardrail_result and aim_output_guardrail_result.get(
"detection_message"
):
@@ -252,7 +294,7 @@ class AimGuardrail(CustomGuardrail):
if aim_output_guardrail_result and aim_output_guardrail_result.get(
"redacted_output"
):
response.choices[0].message.content = aim_output_guardrail_result.get(
choice.message.content = aim_output_guardrail_result.get(
"redacted_output"
)
return response
@@ -254,15 +254,16 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr
) -> Any:
from litellm.types.utils import Choices, ModelResponse
if (
isinstance(response, ModelResponse)
and response.choices
and isinstance(response.choices[0], Choices)
):
content = response.choices[0].message.content or ""
await self.async_make_request(
text=content,
)
if isinstance(response, ModelResponse) and response.choices:
for choice in response.choices:
if not isinstance(choice, Choices):
continue
content = _message_content_to_text(choice.message.content)
if not content:
continue
await self.async_make_request(
text=content,
)
return response
async def async_post_call_streaming_hook(
@@ -279,3 +280,16 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr
error_returned = json.dumps({"error": e.detail})
return f"data: {error_returned}\n\n"
def _message_content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
text_parts = [
item.get("text")
for item in content
if isinstance(item, dict) and isinstance(item.get("text"), str)
]
return "\n".join(part for part in text_parts if part)
return ""
@@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.guardrails.guardrail_hooks.ibm import (
IBMDetectorDetection,
@@ -463,65 +464,53 @@ class IBMGuardrailDetector(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return data
_messages = data.get("messages")
if _messages:
contents_to_check: List[str] = []
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
contents_to_check.append(_content)
# Covers multimodal list content + Responses-API input.
contents_to_check: List[str] = list(iter_message_text(data))
if contents_to_check:
if self.is_detector_server:
# Call detector server with all contents at once
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
if contents_to_check:
if self.is_detector_server:
# Call detector server with all contents at once
result = await self._call_detector_server(
contents=contents_to_check,
verbose_proxy_logger.debug(
"IBM Detector Server async_pre_call_hook result: %s", result
)
# Check if any detections were found
has_violations = False
for message_detections in result:
filtered = self._filter_detections_by_threshold(message_detections)
if filtered:
has_violations = True
break
if has_violations and self.block_on_detection:
error_message = self._create_error_message_detector_server(result)
raise ValueError(error_message)
else:
# Call orchestrator for each content separately
for content in contents_to_check:
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
"IBM Detector Server async_pre_call_hook result: %s", result
"IBM Orchestrator async_pre_call_hook result: %s",
orchestrator_result,
)
# Check if any detections were found
has_violations = False
for message_detections in result:
filtered = self._filter_detections_by_threshold(
message_detections
)
if filtered:
has_violations = True
break
if has_violations and self.block_on_detection:
error_message = self._create_error_message_detector_server(
result
)
raise ValueError(error_message)
else:
# Call orchestrator for each content separately
for content in contents_to_check:
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
"IBM Orchestrator async_pre_call_hook result: %s",
orchestrator_result,
)
filtered = self._filter_detections_by_threshold(
filtered = self._filter_detections_by_threshold(orchestrator_result)
if filtered and self.block_on_detection:
error_message = self._create_error_message_orchestrator(
orchestrator_result
)
if filtered and self.block_on_detection:
error_message = self._create_error_message_orchestrator(
orchestrator_result
)
raise ValueError(error_message)
raise ValueError(error_message)
# Add guardrail to applied guardrails header
add_guardrail_to_applied_guardrails_header(
@@ -550,65 +539,53 @@ class IBMGuardrailDetector(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return
_messages = data.get("messages")
if _messages:
contents_to_check: List[str] = []
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
contents_to_check.append(_content)
# Covers multimodal list content + Responses-API input.
contents_to_check: List[str] = list(iter_message_text(data))
if contents_to_check:
if self.is_detector_server:
# Call detector server with all contents at once
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
if contents_to_check:
if self.is_detector_server:
# Call detector server with all contents at once
result = await self._call_detector_server(
contents=contents_to_check,
verbose_proxy_logger.debug(
"IBM Detector Server async_moderation_hook result: %s", result
)
# Check if any detections were found
has_violations = False
for message_detections in result:
filtered = self._filter_detections_by_threshold(message_detections)
if filtered:
has_violations = True
break
if has_violations and self.block_on_detection:
error_message = self._create_error_message_detector_server(result)
raise ValueError(error_message)
else:
# Call orchestrator for each content separately
for content in contents_to_check:
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
"IBM Detector Server async_moderation_hook result: %s", result
"IBM Orchestrator async_moderation_hook result: %s",
orchestrator_result,
)
# Check if any detections were found
has_violations = False
for message_detections in result:
filtered = self._filter_detections_by_threshold(
message_detections
)
if filtered:
has_violations = True
break
if has_violations and self.block_on_detection:
error_message = self._create_error_message_detector_server(
result
)
raise ValueError(error_message)
else:
# Call orchestrator for each content separately
for content in contents_to_check:
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
"IBM Orchestrator async_moderation_hook result: %s",
orchestrator_result,
)
filtered = self._filter_detections_by_threshold(
filtered = self._filter_detections_by_threshold(orchestrator_result)
if filtered and self.block_on_detection:
error_message = self._create_error_message_orchestrator(
orchestrator_result
)
if filtered and self.block_on_detection:
error_message = self._create_error_message_orchestrator(
orchestrator_result
)
raise ValueError(error_message)
raise ValueError(error_message)
# Add guardrail to applied guardrails header
add_guardrail_to_applied_guardrails_header(
@@ -13,6 +13,11 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
apply_redacted_messages_back,
build_inspection_messages,
has_non_string_content,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.openai import AllMessageValues
@@ -214,18 +219,26 @@ class LakeraAIGuardrail(CustomGuardrail):
)
return data
new_messages: Optional[List[AllMessageValues]] = data.get("messages")
if new_messages is None:
# Covers multimodal list content + Responses-API input.
new_messages = build_inspection_messages(data)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. No messages in data"
"Lakera AI: not running guardrail. No inspectable text in data"
)
return data
# Mask-in-place uses offsets returned by Lakera and can only
# preserve non-text parts (images, audio, …) when the original
# content is a plain string. For multimodal/Responses-API input
# we degrade to block-on-detect so we never silently strip image
# parts while attempting to redact text.
is_multimodal_input = has_non_string_content(data)
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
#########################################################
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
messages=new_messages,
messages=new_messages, # type: ignore[arg-type]
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
@@ -234,13 +247,20 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
# If only PII violations exist, mask the PII
if self._is_only_pii_violation(lakera_guardrail_response):
data["messages"] = self._mask_pii_in_messages(
messages=new_messages,
# If only PII violations exist, mask the PII (string input only).
if (
self._is_only_pii_violation(lakera_guardrail_response)
and not is_multimodal_input
):
redacted_messages = self._mask_pii_in_messages(
messages=new_messages, # type: ignore[arg-type]
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages)) # type: ignore[arg-type]
verbose_proxy_logger.debug(
"Lakera AI: Masked PII in messages instead of blocking request"
)
@@ -252,7 +272,9 @@ class LakeraAIGuardrail(CustomGuardrail):
)
# Log violation but continue
elif self.on_flagged == "block":
# If there are other violations or not set to mask PII, raise exception
# Either non-PII violations, or PII on multimodal input
# (which cannot be masked in place without dropping
# image/audio parts) — raise the standard block error.
raise self._get_http_exception_for_blocked_guardrail(
lakera_guardrail_response
)
@@ -280,18 +302,22 @@ class LakeraAIGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return
new_messages: Optional[List[AllMessageValues]] = data.get("messages")
if new_messages is None:
new_messages = build_inspection_messages(data)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. No messages in data"
"Lakera AI: not running guardrail. No inspectable text in data"
)
return
# See ``async_pre_call_hook`` — multimodal input degrades to
# block-on-detect because mask-in-place would drop image parts.
is_multimodal_input = has_non_string_content(data)
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
#########################################################
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
messages=new_messages,
messages=new_messages, # type: ignore[arg-type]
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
@@ -300,25 +326,28 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
# If only PII violations exist, mask the PII
if self._is_only_pii_violation(lakera_guardrail_response):
data["messages"] = self._mask_pii_in_messages(
messages=new_messages,
if (
self._is_only_pii_violation(lakera_guardrail_response)
and not is_multimodal_input
):
redacted_messages = self._mask_pii_in_messages(
messages=new_messages, # type: ignore[arg-type]
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages)) # type: ignore[arg-type]
verbose_proxy_logger.debug(
"Lakera AI: Masked PII in messages instead of blocking request"
)
else:
# Check on_flagged setting
if self.on_flagged == "monitor":
verbose_proxy_logger.warning(
"Lakera Guardrail: Monitoring mode - violation detected but allowing request"
)
# Log violation but continue
elif self.on_flagged == "block":
# If there are other violations or not set to mask PII, raise exception
raise self._get_http_exception_for_blocked_guardrail(
lakera_guardrail_response
)
@@ -50,6 +50,11 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
apply_redacted_messages_back,
build_inspection_messages,
has_non_string_content,
)
from litellm.types.guardrails import GuardrailEventHooks
import litellm
@@ -366,16 +371,19 @@ class LassoGuardrail(CustomGuardrail):
LassoGuardrailAPIError: If the Lasso API call fails
HTTPException: If blocking violations are detected
"""
messages: List[Dict[str, str]] = data.get("messages", [])
# Covers multimodal list content + Responses-API input.
messages: List[Dict[str, str]] = build_inspection_messages(data)
if not messages:
return data
if self.mask:
# Lasso's classifix endpoint returns masked text that we copy back
# into ``data["messages"]``. For multimodal/Responses-API input we
# would silently strip image/audio parts, so fall back to the
# classify endpoint (which still raises on BLOCK actions) and
# leave the original payload intact.
if self.mask and not has_non_string_content(data):
return await self._handle_masking(data, cache, message_type, messages)
else:
return await self._handle_classification(
data, cache, message_type, messages
)
return await self._handle_classification(data, cache, message_type, messages)
async def _handle_classification(
self,
@@ -413,8 +421,9 @@ class LassoGuardrail(CustomGuardrail):
self._process_lasso_response(response)
# Apply masking to messages if violations detected and masked messages are available
if response.get("violations_detected") and response.get("messages"):
data["messages"] = response["messages"]
redacted_messages = response.get("messages")
if response.get("violations_detected") and redacted_messages:
apply_redacted_messages_back(data, list(redacted_messages))
self._log_masking_applied(message_type, dict(response))
return data
@@ -1873,8 +1873,9 @@ class ContentFilterGuardrail(CustomGuardrail):
and the UI Request Lifecycle panel. Mirrors apply_guardrail's finally-block
contract.
"""
accumulated_full_text = ""
yielded_masked_text_len = 0
accumulated_text_by_choice: Dict[int, str] = {}
yielded_masked_text_len_by_choice: Dict[int, int] = {}
latest_detections_by_choice: Dict[int, List[ContentFilterDetection]] = {}
buffer_size = 50 # Increased buffer to catch patterns split across many chunks
start_time = datetime.now()
@@ -1890,79 +1891,90 @@ class ContentFilterGuardrail(CustomGuardrail):
try:
async for item in response:
if isinstance(item, ModelResponseStream) and item.choices:
delta_content = ""
is_final = False
for choice in item.choices:
if hasattr(choice, "delta") and choice.delta:
content = getattr(choice.delta, "content", None)
if content and isinstance(content, str):
delta_content += content
if getattr(choice, "finish_reason", None):
is_final = True
if not (hasattr(choice, "delta") and choice.delta):
continue
accumulated_full_text += delta_content
choice_index = getattr(choice, "index", 0)
if not isinstance(choice_index, int):
choice_index = 0
# Check for blocking or apply masking
# Add a space at the end if it's the final chunk to trigger word boundaries (\b)
text_to_check = accumulated_full_text
if is_final:
text_to_check += " "
content = getattr(choice.delta, "content", None)
is_final = bool(getattr(choice, "finish_reason", None))
if isinstance(content, str) and content:
accumulated_text_by_choice[choice_index] = (
accumulated_text_by_choice.get(choice_index, "")
+ content
)
elif not is_final:
continue
try:
# Reset before each scan: _filter_single_text scans the
# whole accumulated buffer every chunk, so previous-chunk
# matches are guaranteed to be re-found. Keeping only the
# latest scan's detections avoids N× duplication in the
# final log row. BLOCK still records correctly because
# handlers append to detections before raising.
detections.clear()
masked_text = self._filter_single_text(
text_to_check, detections=detections
text_to_check = accumulated_text_by_choice.get(choice_index, "")
if not text_to_check:
continue
# Add a space at the end if it's the final chunk to trigger word boundaries (\b)
text_to_scan = text_to_check + (" " if is_final else "")
choice_detections: List[ContentFilterDetection] = []
try:
# _filter_single_text scans the whole accumulated
# choice buffer every chunk, so previous-chunk
# matches are guaranteed to be re-found. Keeping
# only each choice's latest scan avoids duplicate
# detections in the final log row.
masked_text = self._filter_single_text(
text_to_scan, detections=choice_detections
)
if is_final and masked_text.endswith(" "):
masked_text = masked_text[:-1]
latest_detections_by_choice[choice_index] = (
choice_detections
)
except HTTPException:
latest_detections_by_choice[choice_index] = (
choice_detections
)
raise
except Exception as e:
verbose_proxy_logger.error(
f"ContentFilterGuardrail: Error in masking: {e}"
)
masked_text = text_to_scan # Fallback to current text
# Determine how much can be safely yielded
if is_final:
safe_to_yield_len = len(masked_text)
else:
safe_to_yield_len = max(0, len(masked_text) - buffer_size)
yielded_masked_text_len = yielded_masked_text_len_by_choice.get(
choice_index, 0
)
if is_final and masked_text.endswith(" "):
masked_text = masked_text[:-1]
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.error(
f"ContentFilterGuardrail: Error in masking: {e}"
)
masked_text = text_to_check # Fallback to current text
if safe_to_yield_len > yielded_masked_text_len:
new_masked_content = masked_text[
yielded_masked_text_len:safe_to_yield_len
]
choice.delta.content = new_masked_content
yielded_masked_text_len_by_choice[choice_index] = (
safe_to_yield_len
)
else:
# Hold content by yielding empty content on this choice
# while preserving chunk metadata and other choices.
choice.delta.content = ""
# Determine how much can be safely yielded
if is_final:
safe_to_yield_len = len(masked_text)
else:
safe_to_yield_len = max(0, len(masked_text) - buffer_size)
if safe_to_yield_len > yielded_masked_text_len:
new_masked_content = masked_text[
yielded_masked_text_len:safe_to_yield_len
]
# Modify the chunk to contain only the new masked content
if (
item.choices
and hasattr(item.choices[0], "delta")
and item.choices[0].delta
):
item.choices[0].delta.content = new_masked_content
yielded_masked_text_len = safe_to_yield_len
yield item
else:
# Hold content by yielding empty content chunk (keeps metadata/structure)
if (
item.choices
and hasattr(item.choices[0], "delta")
and item.choices[0].delta
):
item.choices[0].delta.content = ""
yield item
yield item
else:
# Not a ModelResponseStream or no choices - yield as is
yield item
# Any remaining content (should have been handled by is_final, but just in case)
if yielded_masked_text_len < len(accumulated_full_text):
if any(
yielded_masked_text_len_by_choice.get(choice_index, 0)
< len(accumulated_text)
for choice_index, accumulated_text in accumulated_text_by_choice.items()
):
# We already reached the end of the generator
pass
except HTTPException:
@@ -1973,6 +1985,11 @@ class ContentFilterGuardrail(CustomGuardrail):
exception_str = str(e)
raise e
finally:
detections = [
detection
for choice_detections in latest_detections_by_choice.values()
for detection in choice_detections
]
self._count_masked_entities(detections, masked_entity_count)
self._log_guardrail_information(
request_data=request_data,
@@ -187,11 +187,28 @@ def _extract_user_text(messages: List) -> str:
def _extract_response_text(response: Any) -> str:
"""Extract text from LLM response object."""
"""Extract text from every LLM response choice."""
if hasattr(response, "choices") and response.choices:
choice = response.choices[0]
if hasattr(choice, "message") and choice.message:
return choice.message.content or ""
text_parts: List[str] = []
for choice in response.choices:
if hasattr(choice, "message") and choice.message:
text = _content_to_text(choice.message.content)
if text:
text_parts.append(text)
return "\n".join(text_parts)
return ""
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
text_parts = [
block.get("text")
for block in content
if isinstance(block, dict) and isinstance(block.get("text"), str)
]
return " ".join(part for part in text_parts if part)
return ""
@@ -480,21 +480,32 @@ class XecGuardGuardrail(CustomGuardrail):
choices = response.get("choices")
if not choices:
return None
first = choices[0]
if hasattr(first, "message"):
message = first.message
elif isinstance(first, dict):
message = first.get("message")
text_parts: List[str] = []
for choice in choices:
content = XecGuardGuardrail._extract_choice_content(choice)
text = XecGuardGuardrail._content_to_text(content)
if text:
text_parts.append(text)
return "\n".join(text_parts) or None
@staticmethod
def _extract_choice_content(choice: Any) -> Any:
if hasattr(choice, "message"):
message = choice.message
elif isinstance(choice, dict):
message = choice.get("message")
else:
return None
if message is None:
return None
if hasattr(message, "content"):
content = message.content
elif isinstance(message, dict):
content = message.get("content")
else:
return None
return message.content
if isinstance(message, dict):
return message.get("content")
return None
@staticmethod
def _content_to_text(content: Any) -> Optional[str]:
if isinstance(content, str) and content:
return content
if isinstance(content, list):
+17 -10
View File
@@ -8,6 +8,10 @@ from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
is_text_content_call_type,
iter_message_text,
)
class _PROXY_AzureContentSafety(
@@ -118,10 +122,9 @@ class _PROXY_AzureContentSafety(
):
verbose_proxy_logger.debug("Inside Azure Content-Safety Pre-Call Hook")
try:
if call_type == "completion" and "messages" in data:
for m in data["messages"]:
if "content" in m and isinstance(m["content"], str):
await self.test_violation(content=m["content"], source="input")
if is_text_content_call_type(call_type):
for text in iter_message_text(data):
await self.test_violation(content=text, source="input")
except HTTPException as e:
raise e
@@ -140,12 +143,16 @@ class _PROXY_AzureContentSafety(
response,
):
verbose_proxy_logger.debug("Inside Azure Content-Safety Post-Call Hook")
if isinstance(response, litellm.ModelResponse) and isinstance(
response.choices[0], litellm.utils.Choices
):
await self.test_violation(
content=response.choices[0].message.content or "", source="output"
)
if not isinstance(response, litellm.ModelResponse):
return
for choice in response.choices:
if not isinstance(choice, litellm.utils.Choices):
continue
message = getattr(choice, "message", None)
content = getattr(message, "content", None)
if isinstance(content, str):
await self.test_violation(content=content, source="output")
# async def async_post_call_streaming_hook(
# self,
@@ -171,6 +171,25 @@ class TestHelperFunctions:
mock_response.choices[0].message.content = "Hello from LLM"
assert _extract_response_text(mock_response) == "Hello from LLM"
def test_extract_response_text_combines_all_choices(self):
from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import (
_extract_response_text,
)
first_choice = MagicMock()
first_choice.message.content = "first response"
second_choice = MagicMock()
second_choice.message.content = [
{"type": "text", "text": "second"},
{"type": "text", "text": "response"},
]
mock_response = MagicMock()
mock_response.choices = [first_choice, second_choice]
assert (
_extract_response_text(mock_response) == "first response\nsecond response"
)
def test_extract_response_text_empty(self):
from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import (
_extract_response_text,
@@ -232,6 +232,52 @@ async def test_azure_text_moderation_guardrail_post_call_success_hook():
assert mock_async_make_request.call_args.kwargs["text"] == "Hello world"
@pytest.mark.asyncio
async def test_azure_text_moderation_guardrail_post_call_checks_all_choices():
azure_text_moderation_guardrail = AzureContentSafetyTextModerationGuardrail(
guardrail_name="azure_text_moderation",
api_key="azure_text_moderation_api_key",
api_base="azure_text_moderation_api_base",
)
with patch.object(
azure_text_moderation_guardrail, "async_make_request"
) as mock_async_make_request:
mock_async_make_request.side_effect = [
{
"blocklistsMatch": [],
"categoriesAnalysis": [{"category": "Hate", "severity": 0}],
},
HTTPException(
status_code=400,
detail={"error": "blocked second choice"},
),
]
with pytest.raises(HTTPException):
await azure_text_moderation_guardrail.async_post_call_success_hook(
data={},
user_api_key_dict=UserAPIKeyAuth(
api_key="azure_text_moderation_api_key"
),
response=ModelResponse(
choices=[
Choices(
index=0,
message=Message(content="safe response"),
),
Choices(
index=1,
message=Message(content="unsafe response"),
),
]
),
)
assert [
call.kwargs["text"] for call in mock_async_make_request.call_args_list
] == ["safe response", "unsafe response"]
@pytest.mark.asyncio
async def test_azure_text_moderation_guardrail_post_call_streaming_hook():
@@ -453,6 +453,71 @@ class TestContentFilterGuardrail:
assert "[EMAIL_REDACTED]" in full_content
assert "Contact me at [EMAIL_REDACTED] for info" in full_content
@pytest.mark.asyncio
async def test_streaming_hook_mask_checks_all_choices(self):
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
patterns = [
ContentFilterPattern(
pattern_type="prebuilt",
pattern_name="email",
action=ContentFilterAction.MASK,
),
]
guardrail = ContentFilterGuardrail(
guardrail_name="test-streaming-mask-all-choices",
patterns=patterns,
event_hook=GuardrailEventHooks.during_call,
)
async def mock_stream():
yield ModelResponseStream(
id="chunk1",
choices=[
StreamingChoices(
delta=Delta(content="Contact first@ex"),
index=0,
),
StreamingChoices(
delta=Delta(content="Email second@ex"),
index=1,
),
],
model="gpt-4",
)
yield ModelResponseStream(
id="chunk2",
choices=[
StreamingChoices(
delta=Delta(content="ample.com for help"),
index=0,
finish_reason="stop",
),
StreamingChoices(
delta=Delta(content="ample.com for support"),
index=1,
finish_reason="stop",
),
],
model="gpt-4",
)
content_by_choice = {0: "", 1: ""}
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=MagicMock(),
response=mock_stream(),
request_data={},
):
for choice in chunk.choices:
if choice.delta.content:
content_by_choice[choice.index] += choice.delta.content
assert "first@example.com" not in content_by_choice[0]
assert "second@example.com" not in content_by_choice[1]
assert content_by_choice[0] == "Contact [EMAIL_REDACTED] for help"
assert content_by_choice[1] == "Email [EMAIL_REDACTED] for support"
@pytest.mark.asyncio
async def test_streaming_hook_block(self):
"""
@@ -6,7 +6,6 @@ branch coverage. Network calls are always mocked; the companion live
suite lives in ``test_xecguard_live.py``.
"""
import asyncio
import os
from unittest.mock import MagicMock, patch
@@ -1196,6 +1195,26 @@ class TestXecGuardMessageAssembly:
is None
)
def test_extract_assistant_text_combines_all_choices(self, xecguard_guardrail):
assert (
xecguard_guardrail._extract_assistant_text_from_response(
{
"choices": [
{"message": {"content": "first response"}},
{
"message": {
"content": [
{"type": "text", "text": "second"},
{"type": "text", "text": "response"},
]
}
},
]
}
)
== "first response\nsecond\nresponse"
)
def test_synthesize_user_inputs_not_dict(self, xecguard_guardrail):
assert xecguard_guardrail._synthesize_user_from_inputs("not-dict") is None
@@ -0,0 +1,303 @@
"""Tests for the shared guardrail content extraction helpers."""
from litellm.proxy.guardrails._content_utils import (
apply_redacted_messages_back,
build_inspection_messages,
has_non_string_content,
iter_message_text,
walk_user_text,
)
# ── iter_message_text ────────────────────────────────────────────────────────────
def test_iter_message_text_string_messages():
data = {
"messages": [
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi"},
]
}
assert list(iter_message_text(data)) == ["hello", "hi"]
def test_iter_message_text_multimodal_list_content():
"""VERIA-11: list-format content must be inspected, not silently skipped."""
data = {
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "AWS_KEY=AKIA..."},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "more secrets"},
],
}
]
}
assert list(iter_message_text(data)) == ["AWS_KEY=AKIA...", "more secrets"]
def test_iter_message_text_responses_api_string_input():
"""fniVO9-F: Responses-API ``input`` must be inspectable when ``messages`` absent."""
data = {"input": "tell me a secret"}
assert list(iter_message_text(data)) == ["tell me a secret"]
def test_iter_message_text_responses_api_list_input_messages():
data = {
"input": [
{"role": "user", "content": "first"},
{"role": "user", "content": "second"},
]
}
assert list(iter_message_text(data)) == ["first", "second"]
def test_iter_message_text_responses_api_list_input_content_parts():
data = {
"input": [
{"type": "text", "text": "alpha"},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "beta"},
]
}
assert list(iter_message_text(data)) == ["alpha", "beta"]
def test_iter_message_text_responses_api_list_input_mixed_dicts_and_strings():
"""Greptile P2: mixed-list ``input`` with content-part dicts AND bare
strings must yield every text fragment — read helpers used to truncate
the bare strings."""
data = {
"input": [
{"type": "text", "text": "from-dict"},
"from-bare-string",
{"type": "image_url", "image_url": {"url": "..."}},
"another-bare-string",
]
}
assert list(iter_message_text(data)) == [
"from-dict",
"from-bare-string",
"another-bare-string",
]
def test_iter_message_text_walks_messages_and_input_independently():
"""When both are present (rare), every fragment from either field is
inspected — a stricter guarantee than "first one wins"."""
data = {
"messages": [{"role": "user", "content": "msg-content"}],
"input": "input-content",
}
assert list(iter_message_text(data)) == ["msg-content", "input-content"]
def test_iter_message_text_empty_data():
assert list(iter_message_text({})) == []
assert list(iter_message_text({"messages": []})) == []
assert list(iter_message_text({"input": ""})) == []
# ── walk_user_text ────────────────────────────────────────────────────────────
def test_walk_user_text_redacts_string_messages_in_place():
data = {
"messages": [
{"role": "user", "content": "leak: AKIAEXAMPLE"},
{"role": "assistant", "content": "ok"},
]
}
visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]"))
assert visited == 2
assert data["messages"][0]["content"] == "leak: [REDACTED]"
assert data["messages"][1]["content"] == "ok"
def test_walk_user_text_redacts_multimodal_text_parts():
"""VERIA-11: list-content text parts must be mutable for in-place redaction."""
data = {
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "AKIAEXAMPLE here"},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "no secret"},
],
}
]
}
visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]"))
assert visited == 2
parts = data["messages"][0]["content"]
assert parts[0] == {"type": "text", "text": "[REDACTED] here"}
# Non-text part must be left untouched.
assert parts[1] == {"type": "image_url", "image_url": {"url": "..."}}
assert parts[2] == {"type": "text", "text": "no secret"}
def test_walk_user_text_redacts_responses_api_string_input():
data = {"input": "leak AKIAEXAMPLE"}
visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]"))
assert visited == 1
assert data["input"] == "leak [REDACTED]"
def test_walk_user_text_redacts_responses_api_list_input():
data = {
"input": [
{"type": "text", "text": "AKIAEXAMPLE"},
{"type": "image_url", "image_url": {"url": "..."}},
]
}
visited = walk_user_text(data, lambda s: f"[redacted]{s}[/]")
assert visited == 1
assert data["input"][0] == {"type": "text", "text": "[redacted]AKIAEXAMPLE[/]"}
assert data["input"][1] == {"type": "image_url", "image_url": {"url": "..."}}
def test_walk_user_text_redacts_mixed_list_input():
"""Read and write helpers must agree on coverage — bare strings inside
a mixed ``input`` list are inspected by both."""
data = {
"input": [
{"type": "text", "text": "secret-one"},
"secret-two",
{"type": "image_url", "image_url": {"url": "..."}},
]
}
visited = walk_user_text(data, lambda s: f"<{s}>")
assert visited == 2
assert data["input"][0] == {"type": "text", "text": "<secret-one>"}
assert data["input"][1] == "<secret-two>"
assert data["input"][2] == {"type": "image_url", "image_url": {"url": "..."}}
# ── build_inspection_messages ─────────────────────────────────────────────────
def test_build_inspection_messages_chat_completion_passthrough():
data = {
"messages": [
{"role": "system", "content": "be helpful"},
{"role": "user", "content": "hi"},
]
}
assert build_inspection_messages(data) == [
{"role": "system", "content": "be helpful"},
{"role": "user", "content": "hi"},
]
def test_build_inspection_messages_joins_multimodal_text_parts():
data = {
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "first part"},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "second part"},
],
}
]
}
assert build_inspection_messages(data) == [
{"role": "user", "content": "first part\nsecond part"}
]
def test_build_inspection_messages_lifts_responses_api_input():
"""fniVO9-F: ``input`` must be visible to hooks that POST messages to a remote API."""
data = {"input": "responses-api content"}
assert build_inspection_messages(data) == [
{"role": "user", "content": "responses-api content"}
]
def test_build_inspection_messages_drops_messages_with_no_text():
data = {
"messages": [
{"role": "user", "content": ""},
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "..."}}],
},
{"role": "user", "content": "kept"},
]
}
assert build_inspection_messages(data) == [{"role": "user", "content": "kept"}]
def test_build_inspection_messages_empty_data():
assert build_inspection_messages({}) == []
assert build_inspection_messages({"messages": []}) == []
assert build_inspection_messages({"input": ""}) == []
# ── has_non_string_content ────────────────────────────────────────────────────
def test_has_non_string_content_string_messages():
data = {"messages": [{"role": "user", "content": "hello"}]}
assert has_non_string_content(data) is False
def test_has_non_string_content_multimodal_messages():
data = {"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]}
assert has_non_string_content(data) is True
def test_has_non_string_content_responses_api_string_input():
assert has_non_string_content({"input": "plain string"}) is False
def test_has_non_string_content_responses_api_list_input():
assert has_non_string_content({"input": ["a", "b"]}) is True
def test_has_non_string_content_empty_data():
assert has_non_string_content({}) is False
assert has_non_string_content({"messages": []}) is False
assert has_non_string_content({"input": ""}) is False
# ── apply_redacted_messages_back ──────────────────────────────────────────────
def test_apply_redacted_messages_back_chat_completion():
data = {"messages": [{"role": "user", "content": "secret"}]}
apply_redacted_messages_back(data, [{"role": "user", "content": "[REDACTED]"}])
assert data["messages"] == [{"role": "user", "content": "[REDACTED]"}]
assert "input" not in data
def test_apply_redacted_messages_back_responses_api_string_input():
"""A Responses-API request reads ``data["input"]``; writing only to
``messages`` would let unredacted text reach the LLM."""
data = {"input": "secret payload"}
apply_redacted_messages_back(data, [{"role": "user", "content": "[REDACTED]"}])
assert data["input"] == "[REDACTED]"
def test_apply_redacted_messages_back_both_fields():
"""Defensive: when both fields are present, both are updated."""
data = {
"messages": [{"role": "user", "content": "old"}],
"input": "old",
}
apply_redacted_messages_back(data, [{"role": "user", "content": "[REDACTED]"}])
assert data["messages"] == [{"role": "user", "content": "[REDACTED]"}]
assert data["input"] == "[REDACTED]"
def test_apply_redacted_messages_back_skips_input_when_not_string():
"""List ``input`` (multimodal Responses-API) is left alone — the
multimodal-degrades-to-block guard runs upstream."""
data = {"input": [{"type": "text", "text": "leak"}]}
apply_redacted_messages_back(data, [{"role": "user", "content": "[REDACTED]"}])
assert data["input"] == [{"type": "text", "text": "leak"}]
@@ -0,0 +1,811 @@
"""
Regression tests for guardrail-coverage gaps.
Each test confirms that a previously-bypassable input shape now triggers
inspection by the relevant guardrail hook:
- VERIA-11: multimodal list-format ``content`` is inspected (no longer
silently skipped because of an ``isinstance(content, str)`` check).
- fniVO9-F: Responses-API ``data["input"]`` is inspected (no longer
silently skipped because the hook only looked at ``data["messages"]``).
- yVS0wMDO: Aim's post-call hook inspects every choice when ``n>1``,
not just ``choices[0]``.
"""
from typing import Any, Dict
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from httpx import Request, Response
from litellm import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import Choices, Message, ModelResponse
@pytest.fixture
def user_api_key():
return UserAPIKeyAuth(api_key="hashed", user_id="u", key_alias=None)
# ── Aim ───────────────────────────────────────────────────────────────────────
def _aim_no_action_response() -> Response:
return Response(
status_code=200,
json={"required_action": None},
request=Request("POST", "https://api.aim.security/fw/v1/analyze"),
)
@pytest.mark.asyncio
async def test_aim_inspects_multimodal_list_content(user_api_key, monkeypatch):
monkeypatch.setenv("AIM_API_KEY", "hs-aim-key")
from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail
guard = AimGuardrail()
sent_payload: Dict[str, Any] = {}
async def capture(url, headers, json):
sent_payload.update(json)
return _aim_no_action_response()
with patch.object(guard.async_handler, "post", side_effect=capture):
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "secret payload"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
call_type="acompletion",
)
# The multimodal text part must be visible to Aim.
assert sent_payload["messages"] == [{"role": "user", "content": "secret payload"}]
@pytest.mark.asyncio
async def test_aim_inspects_responses_api_input(user_api_key, monkeypatch):
monkeypatch.setenv("AIM_API_KEY", "hs-aim-key")
from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail
guard = AimGuardrail()
sent_payload: Dict[str, Any] = {}
async def capture(url, headers, json):
sent_payload.update(json)
return _aim_no_action_response()
with patch.object(guard.async_handler, "post", side_effect=capture):
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data={"input": "responses-api content"},
call_type="acompletion",
)
assert sent_payload["messages"] == [
{"role": "user", "content": "responses-api content"}
]
@pytest.mark.asyncio
async def test_aim_post_call_inspects_all_choices(user_api_key, monkeypatch):
"""yVS0wMDO: ``n>1`` no longer bypasses Aim by hiding violations in
``choices[1+]``."""
monkeypatch.setenv("AIM_API_KEY", "hs-aim-key")
from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail
guard = AimGuardrail()
inspected_outputs = []
async def capture(request_data, output, hook, key_alias):
inspected_outputs.append(output)
return {"redacted_output": output}
response = ModelResponse(
choices=[
Choices(index=0, message=Message(role="assistant", content="first")),
Choices(index=1, message=Message(role="assistant", content="second")),
Choices(index=2, message=Message(role="assistant", content="third")),
]
)
with patch.object(guard, "call_aim_guardrail_on_output", side_effect=capture):
await guard.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "hi"}]},
user_api_key_dict=user_api_key,
response=response,
)
# ``asyncio.gather`` is used for parallelism, so order of inspection is
# not guaranteed.
assert sorted(inspected_outputs) == ["first", "second", "third"]
# ── Lakera v2 ─────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_lakera_v2_inspects_responses_api_input(user_api_key, monkeypatch):
monkeypatch.setenv("LAKERA_API_KEY", "lk-test")
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIGuardrail,
)
guard = LakeraAIGuardrail(api_key="lk-test", on_flagged="monitor")
seen_messages = []
async def fake_call_v2_guard(messages, request_data, event_type):
seen_messages.append(messages)
return {"flagged": False}, {}
with patch.object(guard, "call_v2_guard", side_effect=fake_call_v2_guard):
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data={"input": "responses-api content"},
call_type="responses",
)
assert seen_messages == [[{"role": "user", "content": "responses-api content"}]]
@pytest.mark.asyncio
async def test_lakera_v2_responses_api_input_redacted_writeback(
user_api_key, monkeypatch
):
"""Greptile P1: when input arrives via Responses-API ``data["input"]``
(string) and Lakera flags PII, the redacted content must be written
back to ``data["input"]`` — the Responses-API backend reads from
``input``, so writing only to ``messages`` would let unredacted PII
reach the LLM."""
monkeypatch.setenv("LAKERA_API_KEY", "lk-test")
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIGuardrail,
)
guard = LakeraAIGuardrail(api_key="lk-test", on_flagged="block")
async def fake_call_v2_guard(messages, request_data, event_type):
return ({"flagged": True, "payload": []}, {"EMAIL": 1})
def fake_mask(messages, lakera_response, masked_entity_count):
return [{"role": "user", "content": "[REDACTED EMAIL]"}]
with (
patch.object(guard, "call_v2_guard", side_effect=fake_call_v2_guard),
patch.object(guard, "_is_only_pii_violation", return_value=True),
patch.object(guard, "_mask_pii_in_messages", side_effect=fake_mask),
):
data = {"input": "user@example.com leaked"}
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data=data,
call_type="responses",
)
assert data["input"] == "[REDACTED EMAIL]"
@pytest.mark.asyncio
async def test_aim_responses_api_input_anonymize_writeback(user_api_key, monkeypatch):
"""Greptile P1: Aim's anonymize action must redact ``data["input"]``
for Responses-API requests, not just ``data["messages"]``."""
monkeypatch.setenv("AIM_API_KEY", "hs-aim-key")
from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail
guard = AimGuardrail()
aim_response_body = {
"required_action": {"action_type": "anonymize_action"},
"redacted_chat": {
"all_redacted_messages": [
{"role": "user", "content": "[REDACTED] anonymised"}
]
},
}
async def capture(url, headers, json):
return Response(
status_code=200,
json=aim_response_body,
request=Request("POST", "https://api.aim.security/fw/v1/analyze"),
)
with patch.object(guard.async_handler, "post", side_effect=capture):
data = {"input": "user@example.com leaked"}
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data=data,
call_type="responses",
)
assert data["input"] == "[REDACTED] anonymised"
@pytest.mark.asyncio
async def test_lakera_v2_multimodal_pii_degrades_to_block(user_api_key, monkeypatch):
"""Mask-in-place uses Lakera offsets and cannot preserve image/audio
parts of multimodal input. When PII is detected on a multimodal
request, the hook must raise the block exception instead of silently
flattening ``data["messages"]`` to text-only."""
monkeypatch.setenv("LAKERA_API_KEY", "lk-test")
from fastapi import HTTPException
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIGuardrail,
)
guard = LakeraAIGuardrail(api_key="lk-test", on_flagged="block")
async def fake_call_v2_guard(messages, request_data, event_type):
return (
{
"flagged": True,
"payload": [{"detector_type": "pii/email", "start": 0, "end": 5}],
},
{"EMAIL": 1},
)
with (
patch.object(guard, "call_v2_guard", side_effect=fake_call_v2_guard),
patch.object(guard, "_is_only_pii_violation", return_value=True),
patch.object(
guard,
"_get_http_exception_for_blocked_guardrail",
return_value=HTTPException(status_code=400, detail="blocked"),
),
):
with pytest.raises(HTTPException):
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "leak"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
call_type="acompletion",
)
@pytest.mark.asyncio
async def test_lakera_v2_inspects_multimodal_list_content(user_api_key, monkeypatch):
monkeypatch.setenv("LAKERA_API_KEY", "lk-test")
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIGuardrail,
)
guard = LakeraAIGuardrail(api_key="lk-test", on_flagged="monitor")
seen_messages = []
async def fake_call_v2_guard(messages, request_data, event_type):
seen_messages.append(messages)
return {"flagged": False}, {}
with patch.object(guard, "call_v2_guard", side_effect=fake_call_v2_guard):
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "AKIAEXAMPLE"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
call_type="acompletion",
)
assert seen_messages == [[{"role": "user", "content": "AKIAEXAMPLE"}]]
# ── Lasso ─────────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_lasso_multimodal_falls_back_to_classify(user_api_key, monkeypatch):
"""Lasso's classifix (mask) endpoint returns text that overwrites
``data["messages"]``. For multimodal input that would silently strip
image parts — the hook must use the classify endpoint instead and
leave the original payload intact."""
monkeypatch.setenv("LASSO_API_KEY", "ls-test")
from litellm.proxy.guardrails.guardrail_hooks.lasso.lasso import LassoGuardrail
guard = LassoGuardrail(lasso_api_key="ls-test", mask=True)
masking_called = False
classify_called = False
async def fake_masking(data, cache, message_type, messages):
nonlocal masking_called
masking_called = True
return data
async def fake_classification(data, cache, message_type, messages):
nonlocal classify_called
classify_called = True
return data
with (
patch.object(guard, "_handle_masking", side_effect=fake_masking),
patch.object(guard, "_handle_classification", side_effect=fake_classification),
):
await guard._run_lasso_guardrail(
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "hello"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
cache=DualCache(),
message_type="PROMPT",
)
assert classify_called is True
assert masking_called is False
@pytest.mark.asyncio
async def test_lasso_inspects_responses_api_input(user_api_key, monkeypatch):
monkeypatch.setenv("LASSO_API_KEY", "ls-test")
from litellm.proxy.guardrails.guardrail_hooks.lasso.lasso import LassoGuardrail
guard = LassoGuardrail(lasso_api_key="ls-test")
seen_messages = []
async def fake_handle_classification(data, cache, message_type, messages):
seen_messages.append(messages)
return data
with patch.object(
guard, "_handle_classification", side_effect=fake_handle_classification
):
await guard._run_lasso_guardrail(
data={"input": "responses-api content"},
cache=DualCache(),
message_type="PROMPT",
)
assert seen_messages == [[{"role": "user", "content": "responses-api content"}]]
@pytest.mark.asyncio
async def test_lasso_masking_writes_back_responses_api_input(user_api_key, monkeypatch):
"""Krrish blocker: Lasso classifix masking must update ``data["input"]``
for Responses-API requests, not only ``data["messages"]``."""
monkeypatch.setenv("LASSO_API_KEY", "ls-test")
from litellm.proxy.guardrails.guardrail_hooks.lasso.lasso import LassoGuardrail
guard = LassoGuardrail(lasso_api_key="ls-test", mask=True)
lasso_response = {
"violations_detected": True,
"deputies": {"pii": True},
"findings": {"pii": [{"action": "AUTO_MASKING"}]},
"messages": [{"role": "user", "content": "[REDACTED]"}],
}
async def fake_call_lasso_api(headers, payload, api_url=None):
return lasso_response
data = {"input": "user@example.com leaked"}
with patch.object(guard, "_call_lasso_api", side_effect=fake_call_lasso_api):
await guard._run_lasso_guardrail(
data=data,
cache=DualCache(),
message_type="PROMPT",
)
assert data["input"] == "[REDACTED]"
# ── Banned Keywords ───────────────────────────────────────────────────────────
def test_banned_keywords_blocks_multimodal_content(monkeypatch):
"""VERIA-11: a banned word hidden in a multimodal text part is now caught.
Uses ``acompletion`` — the value the proxy ingress actually passes
for ``/v1/chat/completions``. Asserting against the literal sync
``"completion"`` would pass even if the hook's call-type gate were
misaligned with the runtime, so the test wouldn't catch regressions.
"""
monkeypatch.setattr("litellm.banned_keywords_list", ["forbidden"], raising=False)
from enterprise.enterprise_hooks.banned_keywords import _ENTERPRISE_BannedKeywords
from fastapi import HTTPException
guard = _ENTERPRISE_BannedKeywords()
async def _run():
await guard.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_id="u"),
cache=DualCache(),
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "forbidden word here"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
call_type="acompletion",
)
import asyncio
with pytest.raises(HTTPException) as exc:
asyncio.run(_run())
assert "forbidden" in str(exc.value.detail).lower()
def test_banned_keywords_blocks_responses_api_input(monkeypatch):
monkeypatch.setattr("litellm.banned_keywords_list", ["forbidden"], raising=False)
from enterprise.enterprise_hooks.banned_keywords import _ENTERPRISE_BannedKeywords
from fastapi import HTTPException
guard = _ENTERPRISE_BannedKeywords()
async def _run():
await guard.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_id="u"),
cache=DualCache(),
data={"input": "this contains forbidden content"},
call_type="aresponses",
)
import asyncio
with pytest.raises(HTTPException):
asyncio.run(_run())
@pytest.mark.parametrize("call_type", ["completion", "acompletion", "aresponses"])
def test_banned_keywords_fires_on_text_content_call_types(monkeypatch, call_type):
"""Locks the call-type gate to the runtime ``route_type`` values the
proxy actually emits — pinning a regression where the hook had
``call_type == "completion"`` and silently no-op'd both
``acompletion`` (chat completions) and ``aresponses`` (Responses API).
"""
monkeypatch.setattr("litellm.banned_keywords_list", ["forbidden"], raising=False)
from enterprise.enterprise_hooks.banned_keywords import _ENTERPRISE_BannedKeywords
from fastapi import HTTPException
guard = _ENTERPRISE_BannedKeywords()
import asyncio
with pytest.raises(HTTPException):
asyncio.run(
guard.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_id="u"),
cache=DualCache(),
data={
"messages": [{"role": "user", "content": "forbidden text"}],
"input": "forbidden text",
},
call_type=call_type,
)
)
def test_banned_keywords_skips_non_text_call_types(monkeypatch):
"""Embedding / moderation / audio paths don't carry chat text and
aren't in the text-guardrail scope. They must not trigger the hook
even when the request body otherwise looks like a chat payload.
"""
monkeypatch.setattr("litellm.banned_keywords_list", ["forbidden"], raising=False)
from enterprise.enterprise_hooks.banned_keywords import _ENTERPRISE_BannedKeywords
guard = _ENTERPRISE_BannedKeywords()
import asyncio
for call_type in ("aembedding", "amoderation", "aspeech", "atranscription"):
# Should return without raising, even though the data carries the banned word.
asyncio.run(
guard.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_id="u"),
cache=DualCache(),
data={"input": "forbidden text"},
call_type=call_type,
)
)
@pytest.mark.asyncio
async def test_banned_keywords_post_call_checks_all_choices(monkeypatch, user_api_key):
"""Krrish blocker: ``n>1`` responses must not bypass post-call checks by
placing the banned text in ``choices[1+]``."""
monkeypatch.setattr("litellm.banned_keywords_list", ["forbidden"], raising=False)
from enterprise.enterprise_hooks.banned_keywords import _ENTERPRISE_BannedKeywords
from fastapi import HTTPException
guard = _ENTERPRISE_BannedKeywords()
response = ModelResponse(
choices=[
Choices(index=0, message=Message(role="assistant", content="clean")),
Choices(index=1, message=Message(role="assistant", content="forbidden")),
]
)
with pytest.raises(HTTPException) as exc:
await guard.async_post_call_success_hook(
data={},
user_api_key_dict=user_api_key,
response=response,
)
assert "forbidden" in str(exc.value.detail).lower()
# ── Azure Content Safety ──────────────────────────────────────────────────────
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type, data",
[
(
"acompletion",
{
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "scan me"},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
},
),
("aresponses", {"input": "scan me"}),
],
)
async def test_azure_content_safety_pre_call_fires_on_runtime_call_types(
user_api_key, call_type, data
):
"""The proxy ingress passes ``route_type`` straight through as
``call_type`` — ``acompletion`` for chat completions and
``aresponses`` for the Responses API. The hook must inspect text
fragments under both, not only the literal ``"completion"`` string
used by some SDK callers."""
from litellm.proxy.hooks.azure_content_safety import _PROXY_AzureContentSafety
guard = _PROXY_AzureContentSafety.__new__(_PROXY_AzureContentSafety)
seen = []
async def fake_test_violation(content, source=None):
seen.append((content, source))
guard.test_violation = fake_test_violation
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data=data,
call_type=call_type,
)
assert ("scan me", "input") in seen
@pytest.mark.asyncio
async def test_azure_content_safety_post_call_checks_all_choices(user_api_key):
"""Krrish blocker: ``n>1`` responses must not bypass Azure Content Safety
by placing the unsafe text in ``choices[1+]``."""
from fastapi import HTTPException
from litellm.proxy.hooks.azure_content_safety import _PROXY_AzureContentSafety
guard = _PROXY_AzureContentSafety.__new__(_PROXY_AzureContentSafety)
seen_outputs = []
async def fake_test_violation(content, source=None):
seen_outputs.append((content, source))
if "unsafe" in content:
raise HTTPException(status_code=400, detail={"error": "unsafe"})
guard.test_violation = fake_test_violation
response = ModelResponse(
choices=[
Choices(index=0, message=Message(role="assistant", content="clean")),
Choices(index=1, message=Message(role="assistant", content="unsafe")),
Choices(index=2, message=Message(role="assistant", content="later")),
]
)
with pytest.raises(HTTPException):
await guard.async_post_call_success_hook(
data={},
user_api_key_dict=user_api_key,
response=response,
)
assert seen_outputs == [("clean", "output"), ("unsafe", "output")]
# ── Secret Detection ──────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_secret_detection_redacts_multimodal_text_parts(user_api_key):
from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import (
_ENTERPRISE_SecretDetection,
)
guard = _ENTERPRISE_SecretDetection()
data = {
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "AKIAIOSFODNN7EXAMPLE is the key",
},
{"type": "image_url", "image_url": {"url": "..."}},
],
}
]
}
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data=data,
call_type="completion",
)
parts = data["messages"][0]["content"]
assert "AKIAIOSFODNN7EXAMPLE" not in parts[0]["text"]
assert "[REDACTED]" in parts[0]["text"]
# Non-text part is preserved untouched.
assert parts[1] == {"type": "image_url", "image_url": {"url": "..."}}
@pytest.mark.asyncio
async def test_secret_detection_redacts_responses_api_input(user_api_key):
from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import (
_ENTERPRISE_SecretDetection,
)
guard = _ENTERPRISE_SecretDetection()
data = {"input": "leak: AKIAIOSFODNN7EXAMPLE"}
await guard.async_pre_call_hook(
user_api_key_dict=user_api_key,
cache=DualCache(),
data=data,
call_type="moderation",
)
assert "AKIAIOSFODNN7EXAMPLE" not in data["input"]
assert "[REDACTED]" in data["input"]
# ── OpenAI Moderation ─────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_openai_moderation_inspects_multimodal_content(monkeypatch, user_api_key):
"""The aggregated text passed to ``llm_router.amoderation`` must include
list-format text parts and Responses-API input — without this, multimodal
content silently passed moderation."""
from enterprise.enterprise_hooks.openai_moderation import (
_ENTERPRISE_OpenAI_Moderation,
)
guard = _ENTERPRISE_OpenAI_Moderation()
seen_inputs = []
class FakeModeration:
results = [type("R", (), {"flagged": False})()]
async def fake_amoderation(model, input):
seen_inputs.append(input)
return FakeModeration()
fake_router = MagicMock()
fake_router.amoderation = AsyncMock(side_effect=fake_amoderation)
monkeypatch.setattr(
"litellm.proxy.proxy_server.llm_router", fake_router, raising=False
)
await guard.async_moderation_hook(
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "alpha "},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "beta"},
],
}
]
},
user_api_key_dict=user_api_key,
call_type="acompletion",
)
assert seen_inputs == ["alpha beta"]
# ── Google Text Moderation ────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_google_text_moderation_inspects_multimodal_content(user_api_key):
"""The text passed to Google's moderation client must include list-format
text parts."""
from enterprise.enterprise_hooks.google_text_moderation import (
_ENTERPRISE_GoogleTextModeration,
)
guard = _ENTERPRISE_GoogleTextModeration.__new__(_ENTERPRISE_GoogleTextModeration)
seen_documents = []
def fake_language_document(content, type_):
seen_documents.append(content)
return MagicMock()
fake_response = MagicMock()
fake_response.moderation_categories = []
guard.language_document = fake_language_document
guard.moderate_text_request = MagicMock(return_value=MagicMock())
guard.document_type = MagicMock()
guard.client = MagicMock()
guard.client.moderate_text = MagicMock(return_value=fake_response)
await guard.async_moderation_hook(
data={
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "hello "},
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text", "text": "world"},
],
}
]
},
user_api_key_dict=user_api_key,
call_type="acompletion",
)
assert seen_documents == ["hello world"]