mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-01 18:22:53 +00:00
feat(guardrails): LLM-as-a-Judge guardrail (#26360)
* feat(guardrails): add LLM_AS_A_JUDGE to SupportedGuardrailIntegrations * feat(types): add EvalVerdict, StandardLoggingEvalInformation; wire eval_information into SpendLogsMetadata * feat(guardrails): add self-contained llm_as_a_judge guardrail hook * fix(a2a): filter agent-only litellm_params from acompletion kwargs; pass agent_id into body * feat(ui): add LLMJudgeFields criteria builder component * feat(ui): wire LLM-as-a-Judge into add guardrail form * feat(ui): update EvalViewer — title 'LLM Judge Results', weighted score column, summary row * fix(ui): wire EvalViewer into LogDetailContent to show LLM judge results on logs page * fix(guardrails-ui): route llm_as_a_judge to criteria builder step; rename to LiteLLM LLM as a Judge; add litellm logo * fix(guardrail-viewer): stack lifecycle + eval details vertically to avoid badge overflow in narrow drawer * fix(guardrail-create): surface config validation errors on create instead of silently orphaning guardrail in DB * fix(guardrail-registry): hardcode llm_as_a_judge in initializer registry so it loads regardless of package install path * fix(llm-as-a-judge): fix P1 code quality issues - validate weights/on_failure, guard pre_call, handle multimodal, move imports to module level, fix spurious finally logging * fix(guardrail_endpoints): use correct PK field in rollback delete and log rollback failure * fix(llm_as_a_judge): support Pydantic object in _get_litellm_param fallback chain * fix(LLMJudgeFields): replace @tremor/react Button with antd Button * fix(llm_as_a_judge): remove dead registry dicts, fix KeyError in prompt builder, set correct status on judge failure * test(llm_as_a_judge): add unit tests for guardrail hook * fix(llm_as_a_judge): remove @log_guardrail_information decorator to fix duplicate guardrail_information entries The decorator and the manual finally block both called add_standard_logging_guardrail_information_to_request_data, producing two entries per request. The decorator also misclassified HTTPException(422) blocks as guardrail_failed_to_respond (it checks for 400). The finally block correctly tracks status throughout, so removing the decorator is sufficient. * fix(test_gcs_pub_sub): ignore metadata.eval_information in comparison * fix(test_spend_management): ignore metadata.eval_information in payload comparison * fix(types/guardrails): add input_type and messages to ApplyGuardrailRequest * fix(guardrail_endpoints): pass input_type and messages through apply_guardrail endpoint * fix(guardrail_endpoints): auto-detect post_call guardrails and use input_type=response * fix(a2a_endpoints): merge agent litellm_params guardrails into data before post_call hooks * fix(llm_as_a_judge): use float sum with tolerance for weight validation * fix(guardrail_registry): split long import line for black formatting * fix(llm_as_a_judge): guard guardrail_name Optional for mypy * fix(llm_as_a_judge): set guardrail_status=guardrail_intervened when score fails, regardless of on_failure mode * fix(a2a_endpoints): use try/finally so deferred spend log fires even when guardrail blocks with 422 * fix(litellm_logging): declare _defer_async_logging and _enqueue_deferred_logging on Logging class for mypy * fix(logging_worker): restore queue.join() in flush() to wait for in-flight callbacks
This commit is contained in:
ishaan-berri
@@ -20,6 +20,11 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
|
||||
)
|
||||
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
|
||||
|
||||
# Agent metadata fields stored in litellm_params that are not valid litellm.acompletion() kwargs
|
||||
_AGENT_ONLY_PARAMS = frozenset(
|
||||
{"is_public", "agent_name", "agent_id", "agent_card_params"}
|
||||
)
|
||||
|
||||
|
||||
class A2ACompletionBridgeHandler:
|
||||
"""
|
||||
@@ -99,7 +104,7 @@ class A2ACompletionBridgeHandler:
|
||||
litellm_params_to_add = {
|
||||
k: v
|
||||
for k, v in litellm_params.items()
|
||||
if k not in ("model", "custom_llm_provider")
|
||||
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
|
||||
@@ -206,7 +211,7 @@ class A2ACompletionBridgeHandler:
|
||||
litellm_params_to_add = {
|
||||
k: v
|
||||
for k, v in litellm_params.items()
|
||||
if k not in ("model", "custom_llm_provider")
|
||||
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
|
||||
|
||||
@@ -416,6 +416,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
"model": model,
|
||||
}
|
||||
|
||||
# Set by proxy request handlers to defer spend-log fire until after
|
||||
# post_call guardrails have run; the @client decorator then stores the
|
||||
# enqueue closure here instead of firing it immediately.
|
||||
self._defer_async_logging: bool = False
|
||||
self._enqueue_deferred_logging: Optional[Callable[[], None]] = None
|
||||
|
||||
def process_dynamic_callbacks(self):
|
||||
"""
|
||||
Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
|
||||
|
||||
@@ -3348,6 +3348,7 @@ class SpendLogsMetadata(TypedDict):
|
||||
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall]
|
||||
vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]]
|
||||
guardrail_information: Optional[List[StandardLoggingGuardrailInformation]]
|
||||
eval_information: Optional[Any]
|
||||
status: StandardLoggingPayloadStatus
|
||||
proxy_server_request: Optional[str]
|
||||
batch_models: Optional[List[str]]
|
||||
|
||||
@@ -6,7 +6,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
@@ -390,6 +390,7 @@ async def invoke_agent_a2a( # noqa: PLR0915
|
||||
if "metadata" not in body:
|
||||
body["metadata"] = {}
|
||||
body["metadata"]["agent_id"] = agent.agent_id
|
||||
body["agent_id"] = agent.agent_id
|
||||
|
||||
body.update(
|
||||
{
|
||||
@@ -444,6 +445,21 @@ async def invoke_agent_a2a( # noqa: PLR0915
|
||||
static_headers=static_headers or None,
|
||||
)
|
||||
|
||||
# Merge agent-level guardrails into data so post_call_success_hook and
|
||||
# _handle_stream_message both pick them up. A2A agents use model
|
||||
# a2a_agent/*, which is not an llm_router deployment, so
|
||||
# _check_and_merge_model_level_guardrails() skips them.
|
||||
_agent_guardrails = litellm_params.get("guardrails")
|
||||
if _agent_guardrails:
|
||||
if not isinstance(_agent_guardrails, list):
|
||||
_agent_guardrails = [_agent_guardrails]
|
||||
_existing_guardrails: List = data.get("guardrails") or []
|
||||
if not isinstance(_existing_guardrails, list):
|
||||
_existing_guardrails = [_existing_guardrails]
|
||||
data["guardrails"] = _existing_guardrails + [
|
||||
g for g in _agent_guardrails if g not in _existing_guardrails
|
||||
]
|
||||
|
||||
# Route through SDK functions
|
||||
if method == "message/send":
|
||||
from a2a.types import MessageSendParams, SendMessageRequest
|
||||
@@ -452,6 +468,9 @@ async def invoke_agent_a2a( # noqa: PLR0915
|
||||
id=request_id,
|
||||
params=MessageSendParams(**params),
|
||||
)
|
||||
# Defer spend-log until after post_call_success_hook so guardrail
|
||||
# results written by the unified_guardrail hook are captured.
|
||||
logging_obj._defer_async_logging = True # type: ignore[union-attr]
|
||||
response = await asend_message(
|
||||
request=a2a_request,
|
||||
api_base=agent_url,
|
||||
@@ -463,11 +482,18 @@ async def invoke_agent_a2a( # noqa: PLR0915
|
||||
agent_extra_headers=agent_extra_headers,
|
||||
)
|
||||
|
||||
response = await proxy_logging_obj.post_call_success_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=data,
|
||||
response=response,
|
||||
)
|
||||
try:
|
||||
response = await proxy_logging_obj.post_call_success_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=data,
|
||||
response=response,
|
||||
)
|
||||
finally:
|
||||
_enqueue_fn = getattr(logging_obj, "_enqueue_deferred_logging", None)
|
||||
if _enqueue_fn is not None:
|
||||
logging_obj._enqueue_deferred_logging = None # type: ignore[union-attr]
|
||||
_enqueue_fn()
|
||||
|
||||
return JSONResponse(
|
||||
content=(
|
||||
response.model_dump(mode="json", exclude_none=True) # type: ignore
|
||||
|
||||
@@ -7,7 +7,7 @@ import inspect
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast
|
||||
from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
@@ -342,6 +342,21 @@ async def create_guardrail(
|
||||
verbose_proxy_logger.info(
|
||||
f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})"
|
||||
)
|
||||
except (ValueError, TypeError) as init_error:
|
||||
# Configuration error — roll back the DB write so the guardrail isn't orphaned
|
||||
if prisma_client is not None:
|
||||
try:
|
||||
await prisma_client.db.litellm_guardrailstable.delete(
|
||||
where={"guardrail_id": guardrail_id}
|
||||
)
|
||||
except Exception as rollback_err:
|
||||
verbose_proxy_logger.warning(
|
||||
f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Guardrail configuration error: {init_error}",
|
||||
)
|
||||
except Exception as init_error:
|
||||
verbose_proxy_logger.warning(
|
||||
f"Immediate sync: Failed to initialize guardrail '{guardrail_name}' (ID: {guardrail_id}) in memory: {init_error}"
|
||||
@@ -2152,10 +2167,28 @@ async def apply_guardrail(
|
||||
detail=f"Guardrail '{request.guardrail_name}' not found. Please ensure the guardrail is configured in your LiteLLM proxy.",
|
||||
)
|
||||
|
||||
request_data: dict = {}
|
||||
if request.messages:
|
||||
request_data["messages"] = request.messages
|
||||
|
||||
# Auto-detect input_type: if the caller didn't specify "response" but the
|
||||
# guardrail only runs post_call (e.g. LLM-as-a-judge), use "response" so
|
||||
# the test actually exercises the guardrail logic.
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
resolved_input_type = request.input_type
|
||||
if resolved_input_type == "request":
|
||||
hook = getattr(active_guardrail, "event_hook", None)
|
||||
if hook == GuardrailEventHooks.post_call or hook == "post_call":
|
||||
resolved_input_type = "response"
|
||||
|
||||
_input_type: Literal["request", "response"] = (
|
||||
"response" if resolved_input_type == "response" else "request"
|
||||
)
|
||||
guardrailed_inputs = await active_guardrail.apply_guardrail(
|
||||
inputs={"texts": [request.text]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
request_data=request_data,
|
||||
input_type=_input_type,
|
||||
)
|
||||
response_text = guardrailed_inputs.get("texts", [])
|
||||
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
"""LLM-as-a-Judge guardrail: uses an LLM to score responses against weighted criteria."""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
|
||||
|
||||
import litellm
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.guardrails import Guardrail, LitellmParams
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import StandardLoggingEvalInformation
|
||||
|
||||
JUDGE_SYSTEM_PROMPT = """You are a quality judge. Evaluate the assistant's response against the criteria provided.
|
||||
For each criterion, assign a score from 0 to 100 and provide concise reasoning.
|
||||
Return ONLY valid JSON in this exact format:
|
||||
{
|
||||
"verdicts": [
|
||||
{"criterion_name": "<name>", "score": <0-100>, "reasoning": "<one sentence>", "passed": <true|false>, "weight": <weight>}
|
||||
],
|
||||
"overall_score": <weighted average 0-100>
|
||||
}"""
|
||||
|
||||
_VALID_ON_FAILURE = frozenset({"block", "log"})
|
||||
|
||||
|
||||
def _extract_text_from_content(content: Any) -> str:
|
||||
"""Return plain text from a message content field (str or multimodal list)."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
parts.append(part.get("text", ""))
|
||||
return " ".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _get_litellm_param(
|
||||
litellm_params: "LitellmParams",
|
||||
guardrail: "Guardrail",
|
||||
key: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
val = getattr(litellm_params, key, None)
|
||||
if val is not None:
|
||||
return val
|
||||
raw = guardrail.get("litellm_params")
|
||||
if isinstance(raw, dict) and key in raw:
|
||||
return raw[key]
|
||||
if raw is not None and not isinstance(raw, dict):
|
||||
attr = getattr(raw, key, None)
|
||||
if attr is not None:
|
||||
return attr
|
||||
return default
|
||||
|
||||
|
||||
def _build_judge_prompt(
|
||||
criteria: List[Dict[str, Any]],
|
||||
messages: List[Dict[str, Any]],
|
||||
response_text: str,
|
||||
) -> str:
|
||||
criteria_block = "\n".join(
|
||||
f'- {c.get("name", "")} (weight {c.get("weight", 0)}%): {c.get("description", "")}'
|
||||
for c in criteria
|
||||
)
|
||||
conversation = "\n".join(
|
||||
f'{m.get("role", "user").upper()}: {_extract_text_from_content(m.get("content", ""))}'
|
||||
for m in messages
|
||||
if m.get("content") is not None
|
||||
)
|
||||
return (
|
||||
f"Criteria to evaluate:\n{criteria_block}\n\n"
|
||||
f"Conversation:\n{conversation}\n\n"
|
||||
f"Assistant response to evaluate:\n{response_text}"
|
||||
)
|
||||
|
||||
|
||||
class LLMAsAJudgeGuardrail(CustomGuardrail):
|
||||
"""Post-call guardrail that judges response quality via an LLM."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
guardrail_name: str,
|
||||
judge_model: str,
|
||||
criteria: List[Dict[str, Any]],
|
||||
overall_threshold: float = 80.0,
|
||||
on_failure: Literal["block", "log"] = "block",
|
||||
event_hook: Optional[
|
||||
Union[GuardrailEventHooks, List[GuardrailEventHooks]]
|
||||
] = None,
|
||||
default_on: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
_event_hook: Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]] = (
|
||||
None
|
||||
)
|
||||
if event_hook is not None:
|
||||
if isinstance(event_hook, list):
|
||||
_event_hook = [
|
||||
GuardrailEventHooks(h) if isinstance(h, str) else h
|
||||
for h in event_hook
|
||||
]
|
||||
else:
|
||||
_event_hook = (
|
||||
GuardrailEventHooks(event_hook)
|
||||
if isinstance(event_hook, str)
|
||||
else event_hook
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
guardrail_name=guardrail_name,
|
||||
supported_event_hooks=[GuardrailEventHooks.post_call],
|
||||
event_hook=_event_hook or GuardrailEventHooks.post_call,
|
||||
default_on=default_on,
|
||||
**kwargs,
|
||||
)
|
||||
self.judge_model = judge_model
|
||||
self.criteria = criteria
|
||||
self.overall_threshold = overall_threshold
|
||||
self.on_failure = on_failure
|
||||
|
||||
async def _run_judge(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
response_text: str,
|
||||
) -> Dict[str, Any]:
|
||||
judge_messages = [
|
||||
{"role": "system", "content": JUDGE_SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": _build_judge_prompt(self.criteria, messages, response_text),
|
||||
},
|
||||
]
|
||||
response = await litellm.acompletion(
|
||||
model=self.judge_model,
|
||||
messages=judge_messages,
|
||||
response_format={"type": "json_object"},
|
||||
temperature=0,
|
||||
)
|
||||
raw = response.choices[0].message.content or "{}" # type: ignore[union-attr]
|
||||
return json.loads(raw)
|
||||
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict,
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
# Only evaluate post-call (response text). Fail open on pre-call.
|
||||
if input_type != "response":
|
||||
return inputs
|
||||
|
||||
texts = inputs.get("texts") or []
|
||||
response_text = " ".join(texts)
|
||||
if not response_text:
|
||||
return inputs
|
||||
|
||||
start_time = datetime.now()
|
||||
status: GuardrailStatus = "success"
|
||||
judge_result: Dict[str, Any] = {}
|
||||
|
||||
try:
|
||||
messages: List[Dict[str, Any]] = request_data.get("messages") or []
|
||||
|
||||
try:
|
||||
judge_result = await self._run_judge(messages, response_text)
|
||||
except Exception as judge_err:
|
||||
verbose_logger.warning(
|
||||
f"llm_as_a_judge guardrail: judge call failed, failing open. Error: {judge_err}"
|
||||
)
|
||||
status = "guardrail_failed_to_respond"
|
||||
return inputs
|
||||
|
||||
try:
|
||||
overall_score = max(
|
||||
0.0, min(100.0, float(judge_result.get("overall_score", 100)))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
verbose_logger.warning(
|
||||
"llm_as_a_judge: invalid overall_score from judge, failing open"
|
||||
)
|
||||
return inputs
|
||||
|
||||
passed = overall_score >= self.overall_threshold
|
||||
|
||||
eval_info: "StandardLoggingEvalInformation" = {
|
||||
"eval_name": self.guardrail_name or "",
|
||||
"overall_score": overall_score,
|
||||
"passed": passed,
|
||||
"judge_model": self.judge_model,
|
||||
"threshold": self.overall_threshold,
|
||||
"verdicts": judge_result.get("verdicts", []),
|
||||
}
|
||||
_metadata = request_data.setdefault("metadata", {})
|
||||
existing = _metadata.get("eval_information")
|
||||
if isinstance(existing, list):
|
||||
existing.append(eval_info)
|
||||
elif existing is not None:
|
||||
_metadata["eval_information"] = [existing, eval_info]
|
||||
else:
|
||||
_metadata["eval_information"] = eval_info
|
||||
|
||||
if not passed:
|
||||
status = "guardrail_intervened"
|
||||
if self.on_failure == "block":
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={
|
||||
"error": "LLM judge rejected response: score below threshold",
|
||||
"overall_score": overall_score,
|
||||
"threshold": self.overall_threshold,
|
||||
"verdicts": judge_result.get("verdicts", []),
|
||||
},
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.warning(f"llm_as_a_judge guardrail unexpected error: {e}")
|
||||
return inputs
|
||||
finally:
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_provider="llm_as_a_judge",
|
||||
guardrail_json_response=judge_result,
|
||||
request_data=request_data,
|
||||
guardrail_status=status,
|
||||
start_time=start_time.timestamp(),
|
||||
end_time=datetime.now().timestamp(),
|
||||
event_type=GuardrailEventHooks.post_call,
|
||||
)
|
||||
|
||||
|
||||
def initialize_guardrail(
|
||||
litellm_params: "LitellmParams",
|
||||
guardrail: "Guardrail",
|
||||
) -> LLMAsAJudgeGuardrail:
|
||||
guardrail_name = guardrail.get("guardrail_name")
|
||||
if not guardrail_name:
|
||||
raise ValueError("llm_as_a_judge guardrail requires a guardrail_name")
|
||||
|
||||
judge_model = _get_litellm_param(litellm_params, guardrail, "judge_model")
|
||||
if not judge_model:
|
||||
raise ValueError(
|
||||
"llm_as_a_judge guardrail requires judge_model in litellm_params"
|
||||
)
|
||||
|
||||
criteria = _get_litellm_param(litellm_params, guardrail, "criteria") or []
|
||||
if not criteria:
|
||||
raise ValueError("llm_as_a_judge guardrail requires at least one criterion")
|
||||
|
||||
weight_total = sum(float(c.get("weight", 0)) for c in criteria)
|
||||
if abs(weight_total - 100) > 0.5:
|
||||
raise ValueError(
|
||||
f"llm_as_a_judge criterion weights must sum to 100 (got {weight_total})"
|
||||
)
|
||||
|
||||
on_failure = _get_litellm_param(litellm_params, guardrail, "on_failure", "block")
|
||||
if on_failure not in _VALID_ON_FAILURE:
|
||||
raise ValueError(
|
||||
f"llm_as_a_judge on_failure must be 'block' or 'log', got '{on_failure}'"
|
||||
)
|
||||
|
||||
overall_threshold = float(
|
||||
_get_litellm_param(litellm_params, guardrail, "overall_threshold", 80.0)
|
||||
)
|
||||
|
||||
mode = _get_litellm_param(litellm_params, guardrail, "mode")
|
||||
event_hook: Optional[GuardrailEventHooks] = None
|
||||
if isinstance(mode, str) and mode in {e.value for e in GuardrailEventHooks}:
|
||||
event_hook = GuardrailEventHooks(mode)
|
||||
|
||||
instance = LLMAsAJudgeGuardrail(
|
||||
guardrail_name=guardrail_name,
|
||||
judge_model=judge_model,
|
||||
criteria=criteria,
|
||||
overall_threshold=overall_threshold,
|
||||
on_failure=on_failure,
|
||||
event_hook=event_hook,
|
||||
default_on=bool(
|
||||
_get_litellm_param(litellm_params, guardrail, "default_on", False)
|
||||
),
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback(instance)
|
||||
return instance
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LLMAsAJudgeGuardrail",
|
||||
"initialize_guardrail",
|
||||
]
|
||||
@@ -34,6 +34,9 @@ from .guardrail_initializers import (
|
||||
initialize_presidio,
|
||||
initialize_tool_permission,
|
||||
)
|
||||
from .guardrail_hooks.llm_as_a_judge import (
|
||||
initialize_guardrail as initialize_llm_as_a_judge,
|
||||
)
|
||||
|
||||
guardrail_initializer_registry = {
|
||||
SupportedGuardrailIntegrations.BEDROCK.value: initialize_bedrock,
|
||||
@@ -43,6 +46,7 @@ guardrail_initializer_registry = {
|
||||
SupportedGuardrailIntegrations.HIDE_SECRETS.value: initialize_hide_secrets,
|
||||
SupportedGuardrailIntegrations.TOOL_PERMISSION.value: initialize_tool_permission,
|
||||
SupportedGuardrailIntegrations.GRAYSWAN.value: initialize_grayswan,
|
||||
SupportedGuardrailIntegrations.LLM_AS_A_JUDGE.value: initialize_llm_as_a_judge,
|
||||
}
|
||||
|
||||
guardrail_class_registry: Dict[str, Type[CustomGuardrail]] = {
|
||||
|
||||
@@ -107,6 +107,7 @@ def _get_spend_logs_metadata(
|
||||
model_map_information=None,
|
||||
usage_object=None,
|
||||
guardrail_information=None,
|
||||
eval_information=None,
|
||||
cold_storage_object_key=cold_storage_object_key,
|
||||
litellm_overhead_time_ms=None,
|
||||
attempted_retries=None,
|
||||
|
||||
@@ -91,6 +91,7 @@ class SupportedGuardrailIntegrations(Enum):
|
||||
BLOCK_CODE_EXECUTION = "block_code_execution"
|
||||
AKTO = "akto"
|
||||
MCP_JWT_SIGNER = "mcp_jwt_signer"
|
||||
LLM_AS_A_JUDGE = "llm_as_a_judge"
|
||||
|
||||
|
||||
class Role(Enum):
|
||||
@@ -886,6 +887,8 @@ class ApplyGuardrailRequest(BaseModel):
|
||||
text: str
|
||||
language: Optional[str] = None
|
||||
entities: Optional[List[PiiEntityType]] = None
|
||||
input_type: str = "request"
|
||||
messages: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
|
||||
class ApplyGuardrailResponse(BaseModel):
|
||||
|
||||
@@ -2760,6 +2760,29 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
|
||||
"""Risk score 0-10 indicating how risky the request was (higher = riskier). Computed by the guardrail provider."""
|
||||
|
||||
|
||||
class EvalVerdict(TypedDict, total=False):
|
||||
criterion_name: str
|
||||
score: float # 0-100
|
||||
reasoning: str
|
||||
passed: bool
|
||||
weight: int # criterion weight (0-100) as configured in the guardrail
|
||||
|
||||
|
||||
class StandardLoggingEvalInformation(TypedDict, total=False):
|
||||
eval_id: Optional[str]
|
||||
eval_name: str
|
||||
overall_score: float
|
||||
passed: bool
|
||||
judge_model: str
|
||||
iteration: int
|
||||
eval_error: Optional[str]
|
||||
start_time: str
|
||||
end_time: str
|
||||
duration: float
|
||||
verdicts: List[Any]
|
||||
threshold: Optional[float]
|
||||
|
||||
|
||||
class GuardrailTracingDetail(TypedDict, total=False):
|
||||
"""
|
||||
Typed fields for guardrail tracing metadata.
|
||||
|
||||
@@ -42,6 +42,7 @@ ignored_keys = [
|
||||
"metadata.cold_storage_object_key",
|
||||
"metadata.litellm_overhead_time_ms",
|
||||
"metadata.cost_breakdown",
|
||||
"metadata.eval_information",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Unit tests for the LLM-as-a-Judge guardrail hook."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import (
|
||||
LLMAsAJudgeGuardrail,
|
||||
_build_judge_prompt,
|
||||
_extract_text_from_content,
|
||||
initialize_guardrail,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CRITERIA_100 = [
|
||||
{"name": "Accuracy", "weight": 60, "description": "Is it accurate?"},
|
||||
{"name": "Safety", "weight": 40, "description": "Is it safe?"},
|
||||
]
|
||||
|
||||
|
||||
def _make_guardrail(**overrides) -> LLMAsAJudgeGuardrail:
|
||||
kwargs = dict(
|
||||
guardrail_name="test_judge",
|
||||
judge_model="gpt-4o-mini",
|
||||
criteria=CRITERIA_100,
|
||||
overall_threshold=80.0,
|
||||
on_failure="block",
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return LLMAsAJudgeGuardrail(**kwargs)
|
||||
|
||||
|
||||
def _make_verdict_response(overall_score: float) -> dict:
|
||||
return {
|
||||
"verdicts": [
|
||||
{"criterion_name": "Accuracy", "score": overall_score, "reasoning": "ok", "passed": True, "weight": 60},
|
||||
{"criterion_name": "Safety", "score": overall_score, "reasoning": "ok", "passed": True, "weight": 40},
|
||||
],
|
||||
"overall_score": overall_score,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_text_from_content
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_text_str():
|
||||
assert _extract_text_from_content("hello") == "hello"
|
||||
|
||||
|
||||
def test_extract_text_multimodal_list():
|
||||
content = [{"type": "text", "text": "hello"}, {"type": "image_url", "url": "x"}]
|
||||
assert _extract_text_from_content(content) == "hello"
|
||||
|
||||
|
||||
def test_extract_text_unknown_type():
|
||||
assert _extract_text_from_content(42) == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_judge_prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_judge_prompt_contains_criteria():
|
||||
prompt = _build_judge_prompt(CRITERIA_100, [], "response text")
|
||||
assert "Accuracy" in prompt
|
||||
assert "60%" in prompt
|
||||
assert "Safety" in prompt
|
||||
assert "response text" in prompt
|
||||
|
||||
|
||||
def test_build_judge_prompt_missing_name_and_weight():
|
||||
criteria = [{"description": "check it"}]
|
||||
prompt = _build_judge_prompt(criteria, [], "resp")
|
||||
assert "0%" in prompt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# initialize_guardrail — validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_litellm_params(**overrides):
|
||||
params = MagicMock()
|
||||
for attr in ("guardrail_name", "judge_model", "criteria", "on_failure", "overall_threshold", "mode", "default_on"):
|
||||
setattr(params, attr, None)
|
||||
for k, v in overrides.items():
|
||||
setattr(params, k, v)
|
||||
return params
|
||||
|
||||
|
||||
def _make_guardrail_dict(name="g", **litellm_params_overrides):
|
||||
raw = {"judge_model": "gpt-4o-mini", "criteria": CRITERIA_100, "on_failure": "block", "overall_threshold": 80.0}
|
||||
raw.update(litellm_params_overrides)
|
||||
return {"guardrail_name": name, "litellm_params": raw}
|
||||
|
||||
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.logging_callback_manager")
|
||||
def test_initialize_guardrail_ok(mock_mgr):
|
||||
lp = _make_litellm_params()
|
||||
g = _make_guardrail_dict()
|
||||
instance = initialize_guardrail(lp, g)
|
||||
assert isinstance(instance, LLMAsAJudgeGuardrail)
|
||||
mock_mgr.add_litellm_callback.assert_called_once_with(instance)
|
||||
|
||||
|
||||
def test_initialize_guardrail_missing_judge_model():
|
||||
lp = _make_litellm_params()
|
||||
g = _make_guardrail_dict(judge_model=None)
|
||||
g["litellm_params"].pop("judge_model")
|
||||
with pytest.raises(ValueError, match="judge_model"):
|
||||
initialize_guardrail(lp, g)
|
||||
|
||||
|
||||
def test_initialize_guardrail_weight_sum_not_100():
|
||||
lp = _make_litellm_params()
|
||||
bad_criteria = [{"name": "A", "weight": 50, "description": "d"}]
|
||||
g = _make_guardrail_dict(criteria=bad_criteria)
|
||||
with pytest.raises(ValueError, match="100"):
|
||||
initialize_guardrail(lp, g)
|
||||
|
||||
|
||||
def test_initialize_guardrail_invalid_on_failure():
|
||||
lp = _make_litellm_params()
|
||||
g = _make_guardrail_dict(on_failure="explode")
|
||||
with pytest.raises(ValueError, match="on_failure"):
|
||||
initialize_guardrail(lp, g)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_guardrail — enforcement paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_pre_call_passthrough():
|
||||
guardrail = _make_guardrail()
|
||||
inputs = {"texts": ["some text"]}
|
||||
result = await guardrail.apply_guardrail(inputs, {}, "request")
|
||||
assert result is inputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_empty_response_passthrough():
|
||||
guardrail = _make_guardrail()
|
||||
inputs = {"texts": []}
|
||||
result = await guardrail.apply_guardrail(inputs, {}, "response")
|
||||
assert result is inputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion")
|
||||
async def test_apply_guardrail_passes_above_threshold(mock_completion):
|
||||
mock_completion.return_value = MagicMock(
|
||||
choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(90.0))))]
|
||||
)
|
||||
guardrail = _make_guardrail(overall_threshold=80.0)
|
||||
inputs = {"texts": ["good response"]}
|
||||
request_data: dict = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}
|
||||
result = await guardrail.apply_guardrail(inputs, request_data, "response")
|
||||
assert result is inputs
|
||||
assert request_data["metadata"]["eval_information"]["passed"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion")
|
||||
async def test_apply_guardrail_blocks_below_threshold(mock_completion):
|
||||
mock_completion.return_value = MagicMock(
|
||||
choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(50.0))))]
|
||||
)
|
||||
guardrail = _make_guardrail(overall_threshold=80.0, on_failure="block")
|
||||
inputs = {"texts": ["bad response"]}
|
||||
request_data: dict = {"messages": [], "metadata": {}}
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.apply_guardrail(inputs, request_data, "response")
|
||||
assert exc_info.value.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion")
|
||||
async def test_apply_guardrail_log_mode_does_not_block(mock_completion):
|
||||
mock_completion.return_value = MagicMock(
|
||||
choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(50.0))))]
|
||||
)
|
||||
guardrail = _make_guardrail(overall_threshold=80.0, on_failure="log")
|
||||
inputs = {"texts": ["bad response"]}
|
||||
request_data: dict = {"messages": [], "metadata": {}}
|
||||
result = await guardrail.apply_guardrail(inputs, request_data, "response")
|
||||
assert result is inputs
|
||||
assert request_data["metadata"]["eval_information"]["passed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion")
|
||||
async def test_apply_guardrail_judge_error_fails_open(mock_completion):
|
||||
mock_completion.side_effect = RuntimeError("judge down")
|
||||
guardrail = _make_guardrail()
|
||||
inputs = {"texts": ["response"]}
|
||||
request_data: dict = {"messages": [], "metadata": {}}
|
||||
result = await guardrail.apply_guardrail(inputs, request_data, "response")
|
||||
assert result is inputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion")
|
||||
async def test_apply_guardrail_clamps_score(mock_completion):
|
||||
response_payload = {"verdicts": [], "overall_score": 150}
|
||||
mock_completion.return_value = MagicMock(
|
||||
choices=[MagicMock(message=MagicMock(content=json.dumps(response_payload)))]
|
||||
)
|
||||
guardrail = _make_guardrail(overall_threshold=80.0)
|
||||
inputs = {"texts": ["response"]}
|
||||
request_data: dict = {"messages": [], "metadata": {}}
|
||||
result = await guardrail.apply_guardrail(inputs, request_data, "response")
|
||||
assert result is inputs
|
||||
assert request_data["metadata"]["eval_information"]["overall_score"] == 100.0
|
||||
@@ -376,6 +376,7 @@ ignored_keys = [
|
||||
"metadata.error_information",
|
||||
"metadata.attempted_retries",
|
||||
"metadata.max_retries",
|
||||
"metadata.eval_information",
|
||||
]
|
||||
|
||||
MODEL_LIST = [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Form, Input, Modal, Select, Tag, Typography, Button } from "antd";
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import NotificationsManager from "../molecules/notifications_manager";
|
||||
import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings } from "../networking";
|
||||
import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings, modelAvailableCall } from "../networking";
|
||||
import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration";
|
||||
import {
|
||||
choiceToSkipSystemForCreate,
|
||||
@@ -11,10 +11,12 @@ import {
|
||||
populateGuardrailProviderMap,
|
||||
populateGuardrailProviders,
|
||||
shouldRenderContentFilterConfigSettings,
|
||||
shouldRenderLLMJudgeFields,
|
||||
shouldRenderPIIConfigSettings,
|
||||
} from "./guardrail_info_helpers";
|
||||
import GuardrailOptionalParams from "./guardrail_optional_params";
|
||||
import GuardrailProviderFields from "./guardrail_provider_fields";
|
||||
import LLMJudgeFields from "./llm_judge/LLMJudgeFields";
|
||||
import PiiConfiguration from "./pii_configuration";
|
||||
import ToolPermissionRulesEditor, { ToolPermissionConfig } from "./tool_permission/ToolPermissionRulesEditor";
|
||||
|
||||
@@ -126,6 +128,7 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
const [onViolation, setOnViolation] = useState<"warn" | "end_session">("warn");
|
||||
const [realtimeViolationMessage, setRealtimeViolationMessage] = useState<string>("");
|
||||
const [endpointSettingsOpen, setEndpointSettingsOpen] = useState<boolean>(false);
|
||||
const [availableModels, setAvailableModels] = useState<string[]>([]);
|
||||
|
||||
const [toolPermissionConfig, setToolPermissionConfig] = useState<ToolPermissionConfig>({
|
||||
rules: [],
|
||||
@@ -149,13 +152,17 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
// Parallel requests for speed
|
||||
const [uiSettings, providerParamsResp] = await Promise.all([
|
||||
const [uiSettings, providerParamsResp, modelsResp] = await Promise.all([
|
||||
getGuardrailUISettings(accessToken),
|
||||
getGuardrailProviderSpecificParams(accessToken),
|
||||
modelAvailableCall(accessToken, "", "").catch(() => null),
|
||||
]);
|
||||
|
||||
setGuardrailSettings(uiSettings);
|
||||
setProviderParams(providerParamsResp);
|
||||
if (modelsResp?.data) {
|
||||
setAvailableModels(modelsResp.data.map((m: any) => m.id));
|
||||
}
|
||||
|
||||
// Populate dynamic providers from API response
|
||||
populateGuardrailProviders(providerParamsResp);
|
||||
@@ -242,6 +249,11 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
on_disallowed_action: "block",
|
||||
violation_message_template: "",
|
||||
});
|
||||
|
||||
// Default LLM-as-a-Judge to post_call mode
|
||||
if (value === "LlmAsAJudge") {
|
||||
form.setFieldsValue({ mode: "post_call" });
|
||||
}
|
||||
};
|
||||
|
||||
const handleEntitySelect = (entity: string) => {
|
||||
@@ -510,6 +522,29 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
}
|
||||
}
|
||||
|
||||
if (guardrailProvider === "llm_as_a_judge") {
|
||||
const criteria: any[] = values.criteria || [];
|
||||
if (criteria.length === 0) {
|
||||
NotificationsManager.fromBackend("Add at least one evaluation criterion");
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
const weightTotal = criteria.reduce((sum: number, c: any) => sum + (Number(c?.weight) || 0), 0);
|
||||
if (weightTotal !== 100) {
|
||||
NotificationsManager.fromBackend(`Criterion weights must sum to 100% (currently ${weightTotal}%)`);
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
guardrailData.litellm_params.judge_model = values.judge_model;
|
||||
guardrailData.litellm_params.overall_threshold = values.overall_threshold ?? 80;
|
||||
guardrailData.litellm_params.on_failure = values.on_failure ?? "block";
|
||||
guardrailData.litellm_params.criteria = criteria.map((c: any) => ({
|
||||
name: c.name,
|
||||
weight: Number(c.weight),
|
||||
description: c.description || "",
|
||||
}));
|
||||
}
|
||||
|
||||
if (guardrailProvider === "tool_permission") {
|
||||
if (toolPermissionConfig.rules.length === 0) {
|
||||
NotificationsManager.fromBackend("Add at least one tool permission rule");
|
||||
@@ -549,7 +584,8 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
console.log("values: ", JSON.stringify(values));
|
||||
|
||||
// Use pre-fetched provider params to copy recognised params
|
||||
if (providerParams && selectedProvider) {
|
||||
// Skip for providers that handle their own litellm_params (llm_as_a_judge, tool_permission, content filter, PII)
|
||||
if (providerParams && selectedProvider && guardrailProvider !== "llm_as_a_judge") {
|
||||
const providerKey = guardrail_provider_map[selectedProvider]?.toLowerCase();
|
||||
console.log("providerKey: ", providerKey);
|
||||
const providerSpecificParams = providerParams[providerKey] || {};
|
||||
@@ -769,7 +805,7 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
</Form.Item>
|
||||
|
||||
{/* Use the GuardrailProviderFields component to render provider-specific fields */}
|
||||
{!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && (
|
||||
{!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && (
|
||||
<GuardrailProviderFields
|
||||
selectedProvider={selectedProvider}
|
||||
accessToken={accessToken}
|
||||
@@ -875,6 +911,9 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
|
||||
if (shouldRenderContentFilterConfigSettings(selectedProvider)) {
|
||||
return renderContentFilterConfiguration("categories");
|
||||
}
|
||||
if (shouldRenderLLMJudgeFields(selectedProvider)) {
|
||||
return <LLMJudgeFields availableModels={availableModels} form={form} />;
|
||||
}
|
||||
return renderOptionalParams();
|
||||
case 2:
|
||||
if (shouldRenderContentFilterConfigSettings(selectedProvider)) {
|
||||
|
||||
@@ -16,6 +16,7 @@ export const populateGuardrailProviders = (providerParamsResponse: Record<string
|
||||
providers.PresidioPII = "Presidio PII";
|
||||
providers.Bedrock = "Bedrock Guardrail";
|
||||
providers.Lakera = "Lakera";
|
||||
providers.LlmAsAJudge = "LiteLLM LLM as a Judge";
|
||||
|
||||
// Add dynamic providers from API response
|
||||
Object.entries(providerParamsResponse).forEach(([key, value]) => {
|
||||
@@ -49,6 +50,7 @@ export const guardrail_provider_map: Record<string, string> = {
|
||||
ToolPermission: "tool_permission",
|
||||
BlockCodeExecution: "block_code_execution",
|
||||
Promptguard: "promptguard",
|
||||
LlmAsAJudge: "llm_as_a_judge",
|
||||
};
|
||||
|
||||
// Function to populate provider map from API response - updates the original map
|
||||
@@ -103,6 +105,11 @@ export const shouldRenderContentFilterConfigSettings = (provider: string | null)
|
||||
return providerEnum === "LiteLLM Content Filter";
|
||||
};
|
||||
|
||||
export const shouldRenderLLMJudgeFields = (provider: string | null) => {
|
||||
if (!provider) return false;
|
||||
return guardrail_provider_map[provider] === "llm_as_a_judge";
|
||||
};
|
||||
|
||||
const asset_logos_folder = "../ui/assets/logos/";
|
||||
|
||||
export const guardrailLogoMap: Record<string, string> = {
|
||||
@@ -127,6 +134,7 @@ export const guardrailLogoMap: Record<string, string> = {
|
||||
"Prompt Security": `${asset_logos_folder}prompt_security.png`,
|
||||
PromptGuard: `${asset_logos_folder}promptguard.svg`,
|
||||
"LiteLLM Content Filter": `${asset_logos_folder}litellm_logo.jpg`,
|
||||
"LiteLLM LLM as a Judge": `${asset_logos_folder}litellm_logo.jpg`,
|
||||
"Akto": `${asset_logos_folder}akto.svg`,
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import { Form, Select, InputNumber, Input, Tooltip } from "antd";
|
||||
import { PlusOutlined, QuestionCircleOutlined } from "@ant-design/icons";
|
||||
import { Button } from "antd";
|
||||
|
||||
interface LLMJudgeFieldsProps {
|
||||
availableModels: string[];
|
||||
form: any;
|
||||
}
|
||||
|
||||
const LLMJudgeFields: React.FC<LLMJudgeFieldsProps> = ({ availableModels, form }) => {
|
||||
return (
|
||||
<>
|
||||
<div
|
||||
style={{
|
||||
background: "#f6ffed",
|
||||
border: "1px solid #b7eb8f",
|
||||
borderRadius: 6,
|
||||
padding: "10px 14px",
|
||||
marginBottom: 16,
|
||||
fontSize: 13,
|
||||
color: "#389e0d",
|
||||
}}
|
||||
>
|
||||
After each LLM response, the <strong>Judge Model</strong> scores it 0–100 against your criteria.
|
||||
If the weighted average falls below the threshold, the response is blocked (or logged).
|
||||
</div>
|
||||
|
||||
<Form.Item
|
||||
name="judge_model"
|
||||
label={
|
||||
<span>
|
||||
Judge Model
|
||||
<Tooltip title="The LLM that reads each response and grades it. Pick a capable model — it never sees end-user data beyond what the LLM returned.">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
rules={[{ required: true, message: "Select a judge model" }]}
|
||||
>
|
||||
<Select
|
||||
showSearch
|
||||
placeholder="Select a model"
|
||||
options={availableModels.map((m) => ({ label: m, value: m }))}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="overall_threshold"
|
||||
label={
|
||||
<span>
|
||||
Minimum Score to Pass
|
||||
<Tooltip title="0–100. If the weighted average of criterion scores falls below this, the guardrail triggers. 80 is a good default.">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
initialValue={80}
|
||||
>
|
||||
<InputNumber min={0} max={100} addonAfter="/ 100" style={{ width: "100%" }} />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="on_failure"
|
||||
label={
|
||||
<span>
|
||||
On Failure
|
||||
<Tooltip title="Block: return HTTP 422 when the score is too low. Log: record the result but let the response through.">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
initialValue="block"
|
||||
>
|
||||
<Select>
|
||||
<Select.Option value="block">Block (return 422)</Select.Option>
|
||||
<Select.Option value="log">Log only</Select.Option>
|
||||
</Select>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label={
|
||||
<span>
|
||||
Evaluation Criteria
|
||||
<Tooltip title="Each criterion is something the judge checks. Weights must add up to 100%.">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<Form.List name="criteria" initialValue={[{ name: "", weight: 100, description: "" }]}>
|
||||
{(fields, { add, remove }) => (
|
||||
<>
|
||||
{fields.map(({ key, name, ...restField }) => (
|
||||
<div
|
||||
key={key}
|
||||
style={{
|
||||
border: "1px solid #f0f0f0",
|
||||
borderRadius: 6,
|
||||
padding: "12px 12px 0",
|
||||
marginBottom: 8,
|
||||
}}
|
||||
>
|
||||
<div style={{ display: "flex", gap: 8, alignItems: "flex-end" }}>
|
||||
<Form.Item
|
||||
{...restField}
|
||||
name={[name, "name"]}
|
||||
rules={[{ required: true, message: "Enter criterion name" }]}
|
||||
style={{ flex: 2, marginBottom: 8 }}
|
||||
>
|
||||
<Input placeholder="Criterion name (e.g. Policy accuracy)" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
{...restField}
|
||||
name={[name, "weight"]}
|
||||
label={
|
||||
<Tooltip title="How much this criterion counts toward the final score. All weights must add up to 100%.">
|
||||
<span style={{ fontSize: 12, color: "#595959" }}>
|
||||
Weight <QuestionCircleOutlined style={{ color: "#bfbfbf" }} />
|
||||
</span>
|
||||
</Tooltip>
|
||||
}
|
||||
rules={[{ required: true, message: "Enter weight" }]}
|
||||
style={{ flex: 1, marginBottom: 8 }}
|
||||
>
|
||||
<InputNumber
|
||||
min={0}
|
||||
max={100}
|
||||
addonAfter="%"
|
||||
style={{ width: "100%" }}
|
||||
placeholder="e.g. 50"
|
||||
/>
|
||||
</Form.Item>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
size="small"
|
||||
onClick={() => remove(name)}
|
||||
>
|
||||
×
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<Form.Item
|
||||
{...restField}
|
||||
name={[name, "description"]}
|
||||
rules={[{ required: true, message: "Describe what to check" }]}
|
||||
style={{ marginBottom: 8 }}
|
||||
>
|
||||
<Input placeholder="What should the judge check for this criterion?" />
|
||||
</Form.Item>
|
||||
</div>
|
||||
))}
|
||||
<Button
|
||||
type="dashed"
|
||||
block
|
||||
style={{ marginTop: 4 }}
|
||||
onClick={() => add({ name: "", weight: 0, description: "" })}
|
||||
icon={<PlusOutlined />}
|
||||
>
|
||||
Add Criterion
|
||||
</Button>
|
||||
{fields.length > 0 && (
|
||||
<Form.Item shouldUpdate noStyle>
|
||||
{() => {
|
||||
const allCriteria: any[] = form.getFieldValue("criteria") || [];
|
||||
const weightTotal = allCriteria.reduce(
|
||||
(sum: number, c: any) => sum + (Number(c?.weight) || 0),
|
||||
0,
|
||||
);
|
||||
const weightOk = weightTotal === 100;
|
||||
return (
|
||||
<div style={{ marginTop: 6, fontSize: 12, color: weightOk ? "#52c41a" : "#faad14" }}>
|
||||
Weights total: {weightTotal}%{weightOk ? " ✓" : " — must add up to 100%"}
|
||||
</div>
|
||||
);
|
||||
}}
|
||||
</Form.Item>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</Form.List>
|
||||
</Form.Item>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default LLMJudgeFields;
|
||||
@@ -0,0 +1,203 @@
|
||||
import React from "react";
|
||||
import { Card, Tag, Table, Typography, Space, Tooltip } from "antd";
|
||||
import { CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined } from "@ant-design/icons";
|
||||
|
||||
const { Text } = Typography;
|
||||
|
||||
interface EvalVerdict {
|
||||
criterion_name: string;
|
||||
score: number;
|
||||
reasoning: string;
|
||||
passed: boolean;
|
||||
weight?: number;
|
||||
}
|
||||
|
||||
interface EvalInformation {
|
||||
eval_id?: string;
|
||||
eval_name: string;
|
||||
overall_score: number;
|
||||
passed: boolean;
|
||||
judge_model: string;
|
||||
iteration?: number;
|
||||
eval_error?: string | null;
|
||||
verdicts?: EvalVerdict[];
|
||||
threshold?: number;
|
||||
}
|
||||
|
||||
interface EvalViewerProps {
|
||||
data: EvalInformation | EvalInformation[];
|
||||
}
|
||||
|
||||
export default function EvalViewer({ data }: EvalViewerProps) {
|
||||
const entries: EvalInformation[] = Array.isArray(data) ? data : [data];
|
||||
|
||||
if (!entries.length) return null;
|
||||
|
||||
return (
|
||||
<div className="mb-6">
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, marginBottom: 12 }}>
|
||||
<ExperimentOutlined style={{ fontSize: 16, color: "#6366f1" }} />
|
||||
<Text strong style={{ fontSize: 15 }}>
|
||||
LLM Judge Results
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
{entries.map((entry, idx) => (
|
||||
<EvalEntryCard key={entry.eval_id || idx} entry={entry} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function EvalEntryCard({ entry }: { entry: EvalInformation }) {
|
||||
const passed = entry.passed;
|
||||
const scoreColor = passed ? "#52c41a" : "#ff4d4f";
|
||||
|
||||
// Filter out synthetic "Overall" row the judge sometimes appends — it's already in the header
|
||||
const verdicts = (entry.verdicts || []).filter(
|
||||
(v) => (v.criterion_name || "").toLowerCase() !== "overall"
|
||||
);
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: "Criterion",
|
||||
dataIndex: "criterion_name",
|
||||
key: "criterion_name",
|
||||
width: 160,
|
||||
render: (v: string) => <Text strong style={{ whiteSpace: "nowrap" }}>{v}</Text>,
|
||||
},
|
||||
{
|
||||
title: "Weight",
|
||||
dataIndex: "weight",
|
||||
key: "weight",
|
||||
width: 65,
|
||||
render: (v: number) =>
|
||||
v != null ? (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>{v}%</Text>
|
||||
) : null,
|
||||
},
|
||||
{
|
||||
title: "Score",
|
||||
dataIndex: "score",
|
||||
key: "score",
|
||||
width: 65,
|
||||
render: (v: number) => (
|
||||
<Text style={{ color: v >= 70 ? "#52c41a" : v >= 50 ? "#faad14" : "#ff4d4f", fontWeight: 600 }}>
|
||||
{v}
|
||||
</Text>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: (
|
||||
<Tooltip title="Score × Weight — how much each criterion contributes to the final score">
|
||||
<span style={{ borderBottom: "1px dashed #aaa", cursor: "help" }}>Weighted</span>
|
||||
</Tooltip>
|
||||
),
|
||||
key: "weighted",
|
||||
width: 75,
|
||||
render: (_: unknown, row: EvalVerdict) => {
|
||||
if (row.weight == null) return null;
|
||||
const contrib = (row.score * row.weight) / 100;
|
||||
return (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
{contrib % 1 === 0 ? contrib : contrib.toFixed(1)}
|
||||
</Text>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Comment",
|
||||
dataIndex: "reasoning",
|
||||
key: "reasoning",
|
||||
ellipsis: { showTitle: false },
|
||||
render: (v: string) => (
|
||||
<Tooltip title={v}>
|
||||
<span style={{ fontSize: 12 }}>{v}</span>
|
||||
</Tooltip>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Card
|
||||
size="small"
|
||||
className="mb-3"
|
||||
style={{ borderLeft: `3px solid ${scoreColor}` }}
|
||||
title={
|
||||
<Space>
|
||||
{passed ? (
|
||||
<CheckCircleOutlined style={{ color: "#52c41a" }} />
|
||||
) : (
|
||||
<CloseCircleOutlined style={{ color: "#ff4d4f" }} />
|
||||
)}
|
||||
<Text strong>{entry.eval_name}</Text>
|
||||
<Tag color={passed ? "success" : "error"}>{passed ? "PASSED" : "FAILED"}</Tag>
|
||||
<Tooltip title={`Weighted average of all criterion scores. Each criterion has a weight (%) set when the eval was created — higher-weight criteria count more toward the final score.`}>
|
||||
<Text type="secondary" style={{ fontSize: 12, cursor: "help", borderBottom: "1px dashed #aaa" }}>
|
||||
{entry.overall_score?.toFixed(0)} / 100
|
||||
{entry.threshold != null && ` (threshold: ${entry.threshold})`}
|
||||
</Text>
|
||||
</Tooltip>
|
||||
</Space>
|
||||
}
|
||||
extra={
|
||||
<Space size="small">
|
||||
{entry.judge_model && (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
Judge: {entry.judge_model}
|
||||
</Text>
|
||||
)}
|
||||
{entry.iteration != null && (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
Iter: {entry.iteration + 1}
|
||||
</Text>
|
||||
)}
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
{entry.eval_error && (
|
||||
<Text type="warning" style={{ display: "block", marginBottom: 8, fontSize: 12 }}>
|
||||
Judge error: {entry.eval_error}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{verdicts.length > 0 ? (
|
||||
<Table
|
||||
dataSource={verdicts}
|
||||
columns={columns}
|
||||
pagination={false}
|
||||
size="small"
|
||||
rowKey="criterion_name"
|
||||
scroll={{ x: true }}
|
||||
summary={() => {
|
||||
const hasWeights = verdicts.some((v) => v.weight != null);
|
||||
if (!hasWeights) return null;
|
||||
const total = verdicts.reduce(
|
||||
(sum, v) => sum + (v.weight != null ? (v.score * v.weight) / 100 : 0),
|
||||
0
|
||||
);
|
||||
return (
|
||||
<Table.Summary.Row>
|
||||
<Table.Summary.Cell index={0}>
|
||||
<Text strong style={{ fontSize: 12 }}>Total</Text>
|
||||
</Table.Summary.Cell>
|
||||
<Table.Summary.Cell index={1} />
|
||||
<Table.Summary.Cell index={2} />
|
||||
<Table.Summary.Cell index={3}>
|
||||
<Text strong style={{ fontSize: 12, color: scoreColor }}>
|
||||
{total % 1 === 0 ? total : total.toFixed(1)}
|
||||
</Text>
|
||||
</Table.Summary.Cell>
|
||||
<Table.Summary.Cell index={4} />
|
||||
</Table.Summary.Row>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
Score: {entry.overall_score?.toFixed(1)} — no per-criterion breakdown available.
|
||||
</Text>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -733,15 +733,15 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps)
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Body: two columns ──────────────────────────────────── */}
|
||||
<div className="flex">
|
||||
{/* Left column: Request Lifecycle */}
|
||||
<div className="w-[340px] flex-shrink-0 border-r border-gray-100 px-6 py-5">
|
||||
{/* ── Body: stacked ──────────────────────────────────────── */}
|
||||
<div className="flex flex-col">
|
||||
{/* Request Lifecycle */}
|
||||
<div className="border-b border-gray-100 px-6 py-5">
|
||||
<RequestLifecycle entries={guardrailEntries} />
|
||||
</div>
|
||||
|
||||
{/* Right column: Evaluation Details */}
|
||||
<div className="flex-1 px-6 py-5 min-w-0">
|
||||
{/* Evaluation Details */}
|
||||
<div className="px-6 py-5">
|
||||
<h4 className="text-xs font-semibold text-gray-500 uppercase tracking-wider mb-4">
|
||||
Evaluation Details
|
||||
</h4>
|
||||
|
||||
@@ -4,6 +4,7 @@ import moment from "moment";
|
||||
import { LogEntry } from "../columns";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import GuardrailViewer from "../GuardrailViewer/GuardrailViewer";
|
||||
import EvalViewer from "../EvalViewer/EvalViewer";
|
||||
import { CostBreakdownViewer } from "../CostBreakdownViewer";
|
||||
import { ConfigInfoMessage } from "../ConfigInfoMessage";
|
||||
import { VectorStoreViewer } from "../VectorStoreViewer";
|
||||
@@ -68,6 +69,10 @@ export function LogDetailContent({ logEntry, onOpenSettings, isLoadingDetails =
|
||||
const totalMaskedEntities = calculateTotalMaskedEntities(guardrailEntries);
|
||||
const primaryGuardrailLabel = getGuardrailLabel(guardrailEntries);
|
||||
|
||||
// LLM Judge data
|
||||
const evalInfo = metadata?.eval_information;
|
||||
const hasEvalData = evalInfo != null;
|
||||
|
||||
// Vector store data
|
||||
const hasVectorStoreData = checkHasVectorStoreData(metadata);
|
||||
|
||||
@@ -190,6 +195,9 @@ export function LogDetailContent({ logEntry, onOpenSettings, isLoadingDetails =
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* LLM Judge Results */}
|
||||
{hasEvalData && <EvalViewer data={evalInfo} />}
|
||||
|
||||
{/* Vector Store Data */}
|
||||
{hasVectorStoreData && <VectorStoreViewer data={metadata.vector_store_request_metadata} />}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user