Merge pull request #26266 from BerriAI/litellm_bedrock_guardrail_spend_logging_reapply

fix(proxy): Bedrock guardrail spend logs - hook mode, match redaction, streaming request_data
This commit is contained in:
yuneng-jiang
2026-04-22 14:32:10 -07:00
committed by GitHub
9 changed files with 496 additions and 69 deletions
+12
View File
@@ -2,6 +2,7 @@ from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
ClassVar,
Dict,
List,
Literal,
@@ -12,6 +13,7 @@ from typing import (
)
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys
from litellm.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.guardrails import (
@@ -81,6 +83,9 @@ class ModifyResponseException(Exception):
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
def __init__(
self,
guardrail_name: Optional[str] = None,
@@ -637,6 +642,13 @@ class CustomGuardrail(CustomLogger):
if isinstance(item, dict):
item.pop("secret_fields", None)
# Default-safe behavior: never persist raw matched spans in standard
# guardrail logging payloads (single shared implementation; Bedrock hooks pass
# raw provider JSON so redaction is not duplicated upstream).
clean_guardrail_response = redact_nested_match_and_regex_keys(
clean_guardrail_response
)
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,
@@ -1,5 +1,6 @@
# What is this?
## Helper utilities
import copy
from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union
import httpx
@@ -435,3 +436,42 @@ def filter_internal_params(
# Filter out internal parameters
return {k: v for k, v in data.items() if k not in internal_params}
def redact_nested_match_and_regex_keys(
payload: Union[dict, List[Any], str, None],
) -> Union[dict, List[Any], str, None]:
"""
Deep-copy `payload` and replace every `match` / `regex` string field with
"[REDACTED]" anywhere in nested dict/list structures.
Used for guardrail spend/compliance logging so raw spans are not persisted.
"""
if payload is None or isinstance(payload, str):
return payload
try:
redacted: Union[dict, List[Any], str, None] = copy.deepcopy(payload)
except Exception:
return payload
# Iterative traversal; `seen` guards against cyclic refs preserved by deepcopy.
try:
seen: set = set()
stack: List[Any] = [redacted]
while stack:
node = stack.pop()
node_id = id(node)
if node_id in seen:
continue
seen.add(node_id)
if isinstance(node, dict):
if "match" in node:
node["match"] = "[REDACTED]"
if "regex" in node:
node["regex"] = "[REDACTED]"
stack.extend(node.values())
elif isinstance(node, list):
stack.extend(node)
except Exception:
return payload
return redacted
+2 -1
View File
@@ -433,7 +433,8 @@ def add_guardrail_to_applied_guardrails_header(
return
_metadata = request_data.get("metadata", None) or {}
if "applied_guardrails" in _metadata:
_metadata["applied_guardrails"].append(guardrail_name)
if guardrail_name not in _metadata["applied_guardrails"]:
_metadata["applied_guardrails"].append(guardrail_name)
else:
_metadata["applied_guardrails"] = [guardrail_name]
# Ensure metadata is set back to request_data (important when metadata didn't exist)
@@ -5,7 +5,6 @@
# +-------------------------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
import copy
import os
import sys
@@ -18,6 +17,7 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
ClassVar,
Dict,
List,
Literal,
@@ -33,6 +33,7 @@ from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys
from litellm.caching import DualCache
from litellm.exceptions import GuardrailInterventionNormalStringError
from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -79,56 +80,33 @@ class GuardrailMessageFilterResult(NamedTuple):
def _redact_pii_matches(response_json: dict) -> dict:
try:
# Create a deep copy to avoid modifying the original response
redacted_response = copy.deepcopy(response_json)
"""
Redact match-like fields from a Bedrock ApplyGuardrail JSON payload.
# Get assessments from the response
# NOTE: We use `.get("key") or []` instead of `.get("key", [])` because
# the Bedrock API can return explicit `null` for list fields (e.g. "regexes": null).
# In Python, dict.get("key", []) returns None (not []) when the key exists
# with a None/null value. The `or []` ensures we always get an iterable,
# preventing "TypeError: 'NoneType' object is not iterable".
assessments = redacted_response.get("assessments") or []
if not assessments:
return redacted_response
Delegates to :func:`redact_nested_match_and_regex_keys` (same rules as spend
logging). Kept as a Bedrock-module entry point for existing unit tests.
"""
redacted = redact_nested_match_and_regex_keys(response_json)
return redacted if isinstance(redacted, dict) else response_json
for assessment in assessments:
# Redact PII entities in sensitive information policy
sensitive_info_policy = assessment.get("sensitiveInformationPolicy")
if sensitive_info_policy:
pii_entities = sensitive_info_policy.get("piiEntities") or []
for pii_entity in pii_entities:
if "match" in pii_entity:
pii_entity["match"] = "[REDACTED]"
# Redact regex matches
regexes = sensitive_info_policy.get("regexes") or []
for regex_match in regexes:
if "match" in regex_match:
regex_match["match"] = "[REDACTED]"
def _redact_assessment_match_fields(assessments: List[dict]) -> List[dict]:
"""
Redact sensitive match-like fields from blocked assessment summaries.
# Redact custom word matches in word policy
word_policy = assessment.get("wordPolicy")
if word_policy:
custom_words = word_policy.get("customWords") or []
for custom_word in custom_words:
if "match" in custom_word:
custom_word["match"] = "[REDACTED]"
managed_words = word_policy.get("managedWordLists") or []
for managed_word in managed_words:
if "match" in managed_word:
managed_word["match"] = "[REDACTED]"
return redacted_response
except Exception as e:
# We do not want to fail in any case so this is just a warning
verbose_proxy_logger.warning("Guardrail log redaction failed: %s", str(e))
return response_json
This is used for customer-visible error payloads (HTTPException.detail) where
we want to preserve policy/type/action metadata without echoing raw matched
content.
"""
redacted = redact_nested_match_and_regex_keys(assessments)
return redacted if isinstance(redacted, list) else assessments
class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# During-call must use async_moderation_hook (not unified apply_guardrail), otherwise
# OpenAI translation always passes input_type="request" and spend/UI show PRE-CALL.
use_native_during_call_hook: ClassVar[bool] = True
def __init__(
self,
guardrailIdentifier: Optional[str] = None,
@@ -419,6 +397,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
messages: Optional[List[AllMessageValues]] = None,
response: Optional[Union[Any, litellm.ModelResponse]] = None,
request_data: Optional[dict] = None,
logging_event_type: Optional[GuardrailEventHooks] = None,
) -> BedrockGuardrailResponse:
from datetime import datetime
@@ -456,11 +435,17 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
prepared_request.headers,
)
event_type = (
GuardrailEventHooks.pre_call
if source == "INPUT"
else GuardrailEventHooks.post_call
)
# UI / spend logs use event_type. Bedrock's `source` is INPUT vs OUTPUT for the API
# body, which must not be confused with the proxy hook (pre_call / during_call /
# post_call). When omitted, keep legacy mapping for backward compatibility.
if logging_event_type is not None:
event_type = logging_event_type
else:
event_type = (
GuardrailEventHooks.pre_call
if source == "INPUT"
else GuardrailEventHooks.post_call
)
try:
httpx_response = await self.async_handler.post(
@@ -515,9 +500,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
# Add guardrail information to request trace
#########################################################
_json_response = httpx_response.json()
# Raw Bedrock JSON is passed here; match/regex redaction runs once inside
# CustomGuardrail.add_standard_logging_guardrail_information_to_request_data.
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_provider=self.guardrail_provider,
guardrail_json_response=httpx_response.json(),
guardrail_json_response=_json_response,
request_data=request_data or {},
guardrail_status=self._get_bedrock_guardrail_response_status(
response=httpx_response
@@ -530,9 +518,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
if httpx_response.status_code == 200:
# check if the response was flagged
_json_response = httpx_response.json()
redacted_response = _redact_pii_matches(_json_response)
verbose_proxy_logger.debug("Bedrock AI response : %s", redacted_response)
verbose_proxy_logger.debug(
"Bedrock AI response : %s",
redact_nested_match_and_regex_keys(_json_response),
)
bedrock_guardrail_response = BedrockGuardrailResponse(**_json_response)
if self._should_raise_guardrail_blocked_exception(
bedrock_guardrail_response
@@ -809,7 +798,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
assessments = self._extract_blocked_assessments(response)
if assessments:
detail["assessments"] = assessments
detail["assessments"] = _redact_assessment_match_fields(assessments)
return HTTPException(status_code=400, detail=detail)
@@ -831,8 +820,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return False
# Check assessments to determine if any actions were BLOCKED (vs ANONYMIZED)
# NOTE: Use `or []` instead of default param to handle explicit null from Bedrock API.
# See _redact_pii_matches() for detailed explanation of the null safety pattern.
# NOTE: Use `.get("k") or []` not `.get("k", [])` — Bedrock can return explicit
# JSON null; dict.get("k", []) then yields None, and `for x in None` raises.
assessments = response.get("assessments") or []
if not assessments:
return False
@@ -952,7 +941,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
)
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data
source="INPUT",
messages=filtered_messages,
request_data=data,
logging_event_type=GuardrailEventHooks.pre_call,
)
except GuardrailInterventionNormalStringError as e:
bedrock_guardrail_response = e.message
@@ -1024,7 +1016,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
)
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data
source="INPUT",
messages=filtered_messages,
request_data=data,
logging_event_type=GuardrailEventHooks.during_call,
)
except GuardrailInterventionNormalStringError as e:
bedrock_guardrail_response = e.message
@@ -1128,9 +1123,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
source="INPUT",
messages=input_messages,
request_data=data,
logging_event_type=GuardrailEventHooks.post_call,
)
output_task = self.make_bedrock_api_request(
source="OUTPUT", response=response, request_data=data
source="OUTPUT",
response=response,
request_data=data,
logging_event_type=GuardrailEventHooks.post_call,
)
# Execute both requests in parallel
@@ -1144,7 +1143,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# Only run OUTPUT validation (INPUT was already validated in pre_call or during_call)
try:
output_content_bedrock = await self.make_bedrock_api_request(
source="OUTPUT", response=response, request_data=data
source="OUTPUT",
response=response,
request_data=data,
logging_event_type=GuardrailEventHooks.post_call,
)
except GuardrailInterventionNormalStringError as e:
output_content_bedrock = e.message
@@ -1271,9 +1273,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
source="INPUT",
messages=input_messages,
request_data=request_data,
logging_event_type=GuardrailEventHooks.post_call,
) # Only input messages
output_task = self.make_bedrock_api_request(
source="OUTPUT", response=assembled_model_response
source="OUTPUT",
response=assembled_model_response,
request_data=request_data,
logging_event_type=GuardrailEventHooks.post_call,
) # Only response
# Execute both requests in parallel
@@ -1287,7 +1293,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# Only run OUTPUT validation (INPUT was already validated in pre_call or during_call)
try:
output_guardrail_response = await self.make_bedrock_api_request(
source="OUTPUT", response=assembled_model_response
source="OUTPUT",
response=assembled_model_response,
request_data=request_data,
logging_event_type=GuardrailEventHooks.post_call,
)
except GuardrailInterventionNormalStringError as e:
output_guardrail_response = e.message
@@ -1564,6 +1573,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# Bedrock will throw an error if there is no text to process
if filtered_messages:
_log_hook = (
GuardrailEventHooks.pre_call
if input_type == "request"
else GuardrailEventHooks.post_call
)
# Map the abstract input_type to the Bedrock source parameter.
# "request" -> INPUT (scan user-supplied content)
# "response" -> OUTPUT (scan model-generated content)
@@ -1594,12 +1608,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
source="OUTPUT",
response=synthetic_response,
request_data=request_data,
logging_event_type=_log_hook,
)
else:
bedrock_response = await self.make_bedrock_api_request(
source="INPUT",
messages=filtered_messages,
request_data=request_data,
logging_event_type=_log_hook,
)
# Apply any masking that was applied by the guardrail
+6 -1
View File
@@ -940,7 +940,11 @@ class ProxyLogging:
Result from the guardrail execution
"""
# Use unified_guardrail if callback has apply_guardrail method
use_unified = "apply_guardrail" in type(callback).__dict__
has_apply_guardrail = "apply_guardrail" in type(callback).__dict__
use_unified = has_apply_guardrail and not (
hook_type == "during_call"
and getattr(callback, "use_native_during_call_hook", False)
)
if use_unified:
data["guardrail_to_apply"] = callback
@@ -1540,6 +1544,7 @@ class ProxyLogging:
if (
"apply_guardrail" in type(callback).__dict__
and user_api_key_dict is not None
and not getattr(callback, "use_native_during_call_hook", False)
):
data["guardrail_to_apply"] = callback
guardrail_task = self._run_guardrail_task_with_enrichment(
@@ -1107,7 +1107,12 @@ async def test_convert_to_bedrock_format_post_call_streaming_hook():
# Mock the make_bedrock_api_request method to track calls
async def mock_make_bedrock_api_request(
source, messages=None, response=None, request_data=None
source,
messages=None,
response=None,
request_data=None,
logging_event_type=None,
**kwargs,
):
bedrock_calls.append(
{
@@ -1115,6 +1120,7 @@ async def test_convert_to_bedrock_format_post_call_streaming_hook():
"messages": messages,
"response": response,
"request_data": request_data,
"logging_event_type": logging_event_type,
}
)
# Return the mock bedrock response
@@ -1055,3 +1055,50 @@ class TestTracingFieldsPopulation:
assert slg["classification"] == classification
assert slg["detection_method"] == "llm-judge"
assert slg["confidence_score"] == 0.94
class TestCustomGuardrailSpendLogMatchRedaction:
"""Guardrail JSON persisted via standard_logging must not contain raw match spans."""
def test_add_standard_logging_redacts_nested_match(self):
cg = CustomGuardrail(guardrail_name="test-rail")
raw = {
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "match": "GG", "action": "BLOCKED"}
]
}
}
]
}
request_data: dict = {"metadata": {}}
cg.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=raw,
request_data=request_data,
guardrail_status="guardrail_intervened",
)
slg = request_data["metadata"]["standard_logging_guardrail_information"][0]
assert (
slg["guardrail_response"]["assessments"][0]["sensitiveInformationPolicy"][
"piiEntities"
][0]["match"]
== "[REDACTED]"
)
assert raw["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
"match"
] == "GG"
def test_add_standard_logging_redacts_regex_field(self):
cg = CustomGuardrail(guardrail_name="test-rail")
raw = {"filters": [{"regex": r"\d{3}-\d{2}-\d{4}", "action": "BLOCKED"}]}
request_data: dict = {"metadata": {}}
cg.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=raw,
request_data=request_data,
guardrail_status="success",
)
slg = request_data["metadata"]["standard_logging_guardrail_information"][0]
assert slg["guardrail_response"]["filters"][0]["regex"] == "[REDACTED]"
assert raw["filters"][0]["regex"] == r"\d{3}-\d{2}-\d{4}"
@@ -6,6 +6,7 @@ from litellm.litellm_core_utils.core_helpers import (
_FINISH_REASON_MAP,
map_finish_reason,
reconstruct_model_name,
redact_nested_match_and_regex_keys,
)
@@ -158,3 +159,37 @@ class TestFinishReasonMapOutputsAreValid:
f"Mapped value '{openai_reason}' (from '{provider_reason}') "
f"is not a valid OpenAI finish reason"
)
class TestRedactNestedMatchAndRegexKeys:
def test_redacts_match_and_regex_recursively(self):
payload = {
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}
]
},
"wordPolicy": {
"customWords": [{"match": "badword", "action": "BLOCKED"}]
},
}
],
"regex": "should-redact-key-named-regex",
}
out = redact_nested_match_and_regex_keys(payload)
assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][
"match"
] == "[REDACTED]"
assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == (
"[REDACTED]"
)
assert out["regex"] == "[REDACTED]"
assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][
0
]["match"] == "secret-name"
def test_passes_through_none_and_str(self):
assert redact_nested_match_and_regex_keys(None) is None
assert redact_nested_match_and_regex_keys("plain") == "plain"
@@ -12,11 +12,15 @@ from fastapi import HTTPException
sys.path.insert(0, os.path.abspath("../../../../../.."))
import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
BedrockGuardrail,
_redact_pii_matches,
)
from litellm.proxy.utils import ProxyLogging
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import ModelResponse
@@ -106,10 +110,12 @@ async def test__redact_pii_matches_malformed_response():
# Test with completely malformed response
malformed_response = {
"action": "GUARDRAIL_INTERVENED",
"assessments": "not_a_list", # This should cause an exception
# Wrong type for assessments; redact_nested_match_and_regex_keys walks dict
# values and skips non-dict/list nodes, so this must not raise.
"assessments": "not_a_list",
}
# Should not crash and return original response
# Should not crash (deep copy + walk skips the string value under assessments)
redacted_response = _redact_pii_matches(malformed_response)
assert redacted_response == malformed_response
@@ -188,7 +194,7 @@ async def test__redact_pii_matches_multiple_assessments():
@pytest.mark.asyncio
async def test_bedrock_guardrail_logging_uses_redacted_response():
"""Test that the Bedrock guardrail uses redacted response for logging"""
"""Debug logs and standard_logging payloads must not include raw match values."""
# Create proper mock objects
mock_user_api_key_dict = UserAPIKeyAuth()
@@ -295,6 +301,14 @@ async def test_bedrock_guardrail_logging_uses_redacted_response():
== "PHONE"
)
slg_list = request_data["metadata"]["standard_logging_guardrail_information"]
assert (
slg_list[0]["guardrail_response"]["assessments"][0][
"sensitiveInformationPolicy"
]["piiEntities"][0]["match"]
== "[REDACTED]"
)
print("Bedrock guardrail logging redaction test passed")
@@ -1751,6 +1765,124 @@ async def test_bedrock_guardrail_blocked_vs_anonymized_actions():
print("\u2705 BLOCKED vs ANONYMIZED actions test passed")
# ---------------------------------------------------------------------------
# Spend logs: guardrail_mode (pre/during/post) vs Bedrock INPUT/OUTPUT
# ---------------------------------------------------------------------------
def test_bedrock_guardrail_uses_native_during_call_hook():
"""during_call must use async_moderation_hook, not unified apply_guardrail(input=request)."""
assert BedrockGuardrail.use_native_during_call_hook is True
@pytest.mark.asyncio
async def test_make_bedrock_api_request_logging_event_type_for_spend_logs():
"""
Spend/UI use event_type from the proxy hook, not Bedrock's INPUT/OUTPUT alone.
When logging_event_type is set, it must be forwarded to standard guardrail logging.
When omitted, INPUT maps to pre_call (legacy).
"""
guardrail = BedrockGuardrail(
guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT"
)
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_bedrock_response = MagicMock()
mock_bedrock_response.status_code = 200
mock_bedrock_response.json.return_value = {
"action": "NONE",
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "match": "GG", "action": "BLOCKED"}
]
}
}
],
}
request_data = {
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
}
with patch.object(
guardrail.async_handler, "post", new_callable=AsyncMock
) as mock_post, patch.object(
guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")
), patch.object(guardrail, "_prepare_request", return_value=MagicMock()), patch.object(
guardrail,
"add_standard_logging_guardrail_information_to_request_data",
) as mock_log:
mock_post.return_value = mock_bedrock_response
await guardrail.make_bedrock_api_request(
source="INPUT",
messages=request_data["messages"],
request_data=request_data,
logging_event_type=GuardrailEventHooks.during_call,
)
assert mock_log.call_args.kwargs["event_type"] == GuardrailEventHooks.during_call
# Raw Bedrock JSON is forwarded; redaction runs once in
# CustomGuardrail.add_standard_logging_guardrail_information_to_request_data.
assert (
mock_log.call_args.kwargs["guardrail_json_response"]["assessments"][0][
"sensitiveInformationPolicy"
]["piiEntities"][0]["match"]
== "GG"
)
mock_log.reset_mock()
await guardrail.make_bedrock_api_request(
source="INPUT",
messages=request_data["messages"],
request_data=request_data,
)
assert mock_log.call_args.kwargs["event_type"] == GuardrailEventHooks.pre_call
@pytest.mark.asyncio
async def test_during_call_hook_invokes_bedrock_async_moderation_hook():
"""
Bedrock sets use_native_during_call_hook so ProxyLogging runs the real
async_moderation_hook (unified apply_guardrail would log INPUT as pre_call).
"""
cache = DualCache()
proxy_logging = ProxyLogging(user_api_key_cache=cache)
guardrail = BedrockGuardrail(
guardrail_name="bedrock-during-test",
guardrailIdentifier="gid",
guardrailVersion="1",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_mod = AsyncMock(return_value=None)
original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
try:
litellm.callbacks = [guardrail]
with patch.object(guardrail, "async_moderation_hook", new=mock_mod):
await proxy_logging.during_call_hook(
data={
"model": "gpt-4",
"messages": [{"role": "user", "content": "test"}],
},
user_api_key_dict=UserAPIKeyAuth(
api_key="test_key", user_id="test_user"
),
call_type="completion",
)
finally:
litellm.callbacks = original_callbacks
mock_mod.assert_awaited_once()
# ---------------------------------------------------------------------------
# L3: _extract_blocked_assessments + _get_http_exception_for_blocked_guardrail
# Regression coverage for case 2026-04-10-internal-bedrock-guardrail-streaming-error.
@@ -1766,7 +1898,7 @@ def _make_guardrail() -> BedrockGuardrail:
def test_extract_blocked_assessments_pii_entity():
"""L3: PII entity match (BLOCKED) is surfaced with category, type, and matched term."""
"""L3: PII entity match (BLOCKED) is surfaced with category, type, and match."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
@@ -1877,6 +2009,7 @@ def test_get_http_exception_includes_assessments_and_identifier():
assert exc.detail["guardrailVersion"] == "1"
assert exc.detail["assessments"][0]["policy"] == "sensitiveInformationPolicy"
assert exc.detail["assessments"][0]["matches"][0]["type"] == "NAME"
assert exc.detail["assessments"][0]["matches"][0]["match"] == "[REDACTED]"
def test_get_http_exception_no_blocked_assessments_omits_field():
@@ -1899,3 +2032,135 @@ def test_get_http_exception_no_blocked_assessments_omits_field():
assert isinstance(exc, HTTPException)
assert "assessments" not in exc.detail
assert exc.detail["guardrailIdentifier"] == "amgllac6xf3r"
@pytest.mark.asyncio
async def test_streaming_post_call_parallel_output_passes_request_data_to_make_bedrock():
"""
async_post_call_streaming_iterator_hook must pass request_data into OUTPUT
make_bedrock_api_request so spend/standard_logging attaches to the real request
(Greptile: previously OUTPUT used request_data=None / ephemeral {}).
"""
request_data = {
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "hi"}],
"metadata": {"stream_guardrail_logging": True},
}
guardrail = BedrockGuardrail(
guardrail_name="bedrock-stream-reqdata",
guardrailIdentifier="test-id",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.post_call,
default_on=True,
)
mock_chunks = [
litellm.ModelResponseStream(
id="tid",
choices=[
litellm.types.utils.StreamingChoices(
delta=litellm.types.utils.Delta(content="Hi", role="assistant"),
finish_reason=None,
index=0,
)
],
created=1,
model="gpt-4o-mini",
object="chat.completion.chunk",
),
litellm.ModelResponseStream(
id="tid",
choices=[
litellm.types.utils.StreamingChoices(
delta=litellm.types.utils.Delta(content="!", role="assistant"),
finish_reason="stop",
index=0,
)
],
created=1,
model="gpt-4o-mini",
object="chat.completion.chunk",
),
]
async def mock_stream():
for c in mock_chunks:
yield c
minimal = {"action": "NONE", "assessments": [], "outputs": []}
with patch.object(
guardrail, "make_bedrock_api_request", AsyncMock(return_value=minimal)
) as mock_make:
out = []
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(),
response=mock_stream(),
request_data=request_data,
):
out.append(chunk)
assert len(out) >= 1
output_calls = [
c for c in mock_make.call_args_list if c.kwargs.get("source") == "OUTPUT"
]
assert len(output_calls) == 1
assert output_calls[0].kwargs.get("request_data") is request_data
assert (
output_calls[0].kwargs.get("logging_event_type")
== GuardrailEventHooks.post_call
)
input_calls = [
c for c in mock_make.call_args_list if c.kwargs.get("source") == "INPUT"
]
assert len(input_calls) == 1
assert input_calls[0].kwargs.get("request_data") is request_data
@pytest.mark.asyncio
async def test_streaming_post_call_output_only_path_passes_request_data_to_make_bedrock():
"""When INPUT validation is skipped (pre/during already ran), OUTPUT still gets request_data."""
request_data = {
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "hi"}],
}
guardrail = BedrockGuardrail(
guardrail_name="bedrock-stream-out-only",
guardrailIdentifier="test-id",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_chunks = [
litellm.ModelResponseStream(
id="tid",
choices=[
litellm.types.utils.StreamingChoices(
delta=litellm.types.utils.Delta(content="x", role="assistant"),
finish_reason="stop",
index=0,
)
],
created=1,
model="gpt-4o-mini",
object="chat.completion.chunk",
),
]
async def mock_stream():
for c in mock_chunks:
yield c
minimal = {"action": "NONE", "assessments": [], "outputs": []}
with patch.object(
guardrail, "make_bedrock_api_request", AsyncMock(return_value=minimal)
) as mock_make:
async for _ in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(),
response=mock_stream(),
request_data=request_data,
):
pass
assert mock_make.call_count == 1
c = mock_make.call_args
assert c.kwargs.get("source") == "OUTPUT"
assert c.kwargs.get("request_data") is request_data