diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 53aac1d3e6..4e66fe4ba6 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -20,6 +20,11 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import ( ) from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager +# Agent metadata fields stored in litellm_params that are not valid litellm.acompletion() kwargs +_AGENT_ONLY_PARAMS = frozenset( + {"is_public", "agent_name", "agent_id", "agent_card_params"} +) + class A2ACompletionBridgeHandler: """ @@ -99,7 +104,7 @@ class A2ACompletionBridgeHandler: litellm_params_to_add = { k: v for k, v in litellm_params.items() - if k not in ("model", "custom_llm_provider") + if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS } completion_params.update(litellm_params_to_add) @@ -206,7 +211,7 @@ class A2ACompletionBridgeHandler: litellm_params_to_add = { k: v for k, v in litellm_params.items() - if k not in ("model", "custom_llm_provider") + if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS } completion_params.update(litellm_params_to_add) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 625cb83724..9e333441c3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -416,6 +416,12 @@ class Logging(LiteLLMLoggingBaseClass): "model": model, } + # Set by proxy request handlers to defer spend-log fire until after + # post_call guardrails have run; the @client decorator then stores the + # enqueue closure here instead of firing it immediately. + self._defer_async_logging: bool = False + self._enqueue_deferred_logging: Optional[Callable[[], None]] = None + def process_dynamic_callbacks(self): """ Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b8e0aa0d12..08b3238bc6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3348,6 +3348,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]] guardrail_information: Optional[List[StandardLoggingGuardrailInformation]] + eval_information: Optional[Any] status: StandardLoggingPayloadStatus proxy_server_request: Optional[str] batch_models: Optional[List[str]] diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 9b18c76629..72a34f6157 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -6,7 +6,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM """ import json -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse, StreamingResponse @@ -390,6 +390,7 @@ async def invoke_agent_a2a( # noqa: PLR0915 if "metadata" not in body: body["metadata"] = {} body["metadata"]["agent_id"] = agent.agent_id + body["agent_id"] = agent.agent_id body.update( { @@ -444,6 +445,21 @@ async def invoke_agent_a2a( # noqa: PLR0915 static_headers=static_headers or None, ) + # Merge agent-level guardrails into data so post_call_success_hook and + # _handle_stream_message both pick them up. A2A agents use model + # a2a_agent/*, which is not an llm_router deployment, so + # _check_and_merge_model_level_guardrails() skips them. + _agent_guardrails = litellm_params.get("guardrails") + if _agent_guardrails: + if not isinstance(_agent_guardrails, list): + _agent_guardrails = [_agent_guardrails] + _existing_guardrails: List = data.get("guardrails") or [] + if not isinstance(_existing_guardrails, list): + _existing_guardrails = [_existing_guardrails] + data["guardrails"] = _existing_guardrails + [ + g for g in _agent_guardrails if g not in _existing_guardrails + ] + # Route through SDK functions if method == "message/send": from a2a.types import MessageSendParams, SendMessageRequest @@ -452,6 +468,9 @@ async def invoke_agent_a2a( # noqa: PLR0915 id=request_id, params=MessageSendParams(**params), ) + # Defer spend-log until after post_call_success_hook so guardrail + # results written by the unified_guardrail hook are captured. + logging_obj._defer_async_logging = True # type: ignore[union-attr] response = await asend_message( request=a2a_request, api_base=agent_url, @@ -463,11 +482,18 @@ async def invoke_agent_a2a( # noqa: PLR0915 agent_extra_headers=agent_extra_headers, ) - response = await proxy_logging_obj.post_call_success_hook( - user_api_key_dict=user_api_key_dict, - data=data, - response=response, - ) + try: + response = await proxy_logging_obj.post_call_success_hook( + user_api_key_dict=user_api_key_dict, + data=data, + response=response, + ) + finally: + _enqueue_fn = getattr(logging_obj, "_enqueue_deferred_logging", None) + if _enqueue_fn is not None: + logging_obj._enqueue_deferred_logging = None # type: ignore[union-attr] + _enqueue_fn() + return JSONResponse( content=( response.model_dump(mode="json", exclude_none=True) # type: ignore diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index b38b5bb1f6..d8c89456d1 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -7,7 +7,7 @@ import inspect import json import os from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException @@ -342,6 +342,21 @@ async def create_guardrail( verbose_proxy_logger.info( f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})" ) + except (ValueError, TypeError) as init_error: + # Configuration error — roll back the DB write so the guardrail isn't orphaned + if prisma_client is not None: + try: + await prisma_client.db.litellm_guardrailstable.delete( + where={"guardrail_id": guardrail_id} + ) + except Exception as rollback_err: + verbose_proxy_logger.warning( + f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}" + ) + raise HTTPException( + status_code=400, + detail=f"Guardrail configuration error: {init_error}", + ) except Exception as init_error: verbose_proxy_logger.warning( f"Immediate sync: Failed to initialize guardrail '{guardrail_name}' (ID: {guardrail_id}) in memory: {init_error}" @@ -2152,10 +2167,28 @@ async def apply_guardrail( detail=f"Guardrail '{request.guardrail_name}' not found. Please ensure the guardrail is configured in your LiteLLM proxy.", ) + request_data: dict = {} + if request.messages: + request_data["messages"] = request.messages + + # Auto-detect input_type: if the caller didn't specify "response" but the + # guardrail only runs post_call (e.g. LLM-as-a-judge), use "response" so + # the test actually exercises the guardrail logic. + from litellm.types.guardrails import GuardrailEventHooks + + resolved_input_type = request.input_type + if resolved_input_type == "request": + hook = getattr(active_guardrail, "event_hook", None) + if hook == GuardrailEventHooks.post_call or hook == "post_call": + resolved_input_type = "response" + + _input_type: Literal["request", "response"] = ( + "response" if resolved_input_type == "response" else "request" + ) guardrailed_inputs = await active_guardrail.apply_guardrail( inputs={"texts": [request.text]}, - request_data={}, - input_type="request", + request_data=request_data, + input_type=_input_type, ) response_text = guardrailed_inputs.get("texts", []) diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py new file mode 100644 index 0000000000..45bdd6dd09 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -0,0 +1,301 @@ +"""LLM-as-a-Judge guardrail: uses an LLM to score responses against weighted criteria.""" + +import json +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union + +import litellm +from fastapi import HTTPException + +from litellm._logging import verbose_logger +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import StandardLoggingEvalInformation + +JUDGE_SYSTEM_PROMPT = """You are a quality judge. Evaluate the assistant's response against the criteria provided. +For each criterion, assign a score from 0 to 100 and provide concise reasoning. +Return ONLY valid JSON in this exact format: +{ + "verdicts": [ + {"criterion_name": "", "score": <0-100>, "reasoning": "", "passed": , "weight": } + ], + "overall_score": +}""" + +_VALID_ON_FAILURE = frozenset({"block", "log"}) + + +def _extract_text_from_content(content: Any) -> str: + """Return plain text from a message content field (str or multimodal list).""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(part.get("text", "")) + return " ".join(parts) + return "" + + +def _get_litellm_param( + litellm_params: "LitellmParams", + guardrail: "Guardrail", + key: str, + default: Any = None, +) -> Any: + val = getattr(litellm_params, key, None) + if val is not None: + return val + raw = guardrail.get("litellm_params") + if isinstance(raw, dict) and key in raw: + return raw[key] + if raw is not None and not isinstance(raw, dict): + attr = getattr(raw, key, None) + if attr is not None: + return attr + return default + + +def _build_judge_prompt( + criteria: List[Dict[str, Any]], + messages: List[Dict[str, Any]], + response_text: str, +) -> str: + criteria_block = "\n".join( + f'- {c.get("name", "")} (weight {c.get("weight", 0)}%): {c.get("description", "")}' + for c in criteria + ) + conversation = "\n".join( + f'{m.get("role", "user").upper()}: {_extract_text_from_content(m.get("content", ""))}' + for m in messages + if m.get("content") is not None + ) + return ( + f"Criteria to evaluate:\n{criteria_block}\n\n" + f"Conversation:\n{conversation}\n\n" + f"Assistant response to evaluate:\n{response_text}" + ) + + +class LLMAsAJudgeGuardrail(CustomGuardrail): + """Post-call guardrail that judges response quality via an LLM.""" + + def __init__( + self, + guardrail_name: str, + judge_model: str, + criteria: List[Dict[str, Any]], + overall_threshold: float = 80.0, + on_failure: Literal["block", "log"] = "block", + event_hook: Optional[ + Union[GuardrailEventHooks, List[GuardrailEventHooks]] + ] = None, + default_on: bool = False, + **kwargs: Any, + ) -> None: + _event_hook: Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]] = ( + None + ) + if event_hook is not None: + if isinstance(event_hook, list): + _event_hook = [ + GuardrailEventHooks(h) if isinstance(h, str) else h + for h in event_hook + ] + else: + _event_hook = ( + GuardrailEventHooks(event_hook) + if isinstance(event_hook, str) + else event_hook + ) + + super().__init__( + guardrail_name=guardrail_name, + supported_event_hooks=[GuardrailEventHooks.post_call], + event_hook=_event_hook or GuardrailEventHooks.post_call, + default_on=default_on, + **kwargs, + ) + self.judge_model = judge_model + self.criteria = criteria + self.overall_threshold = overall_threshold + self.on_failure = on_failure + + async def _run_judge( + self, + messages: List[Dict[str, Any]], + response_text: str, + ) -> Dict[str, Any]: + judge_messages = [ + {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, + { + "role": "user", + "content": _build_judge_prompt(self.criteria, messages, response_text), + }, + ] + response = await litellm.acompletion( + model=self.judge_model, + messages=judge_messages, + response_format={"type": "json_object"}, + temperature=0, + ) + raw = response.choices[0].message.content or "{}" # type: ignore[union-attr] + return json.loads(raw) + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + # Only evaluate post-call (response text). Fail open on pre-call. + if input_type != "response": + return inputs + + texts = inputs.get("texts") or [] + response_text = " ".join(texts) + if not response_text: + return inputs + + start_time = datetime.now() + status: GuardrailStatus = "success" + judge_result: Dict[str, Any] = {} + + try: + messages: List[Dict[str, Any]] = request_data.get("messages") or [] + + try: + judge_result = await self._run_judge(messages, response_text) + except Exception as judge_err: + verbose_logger.warning( + f"llm_as_a_judge guardrail: judge call failed, failing open. Error: {judge_err}" + ) + status = "guardrail_failed_to_respond" + return inputs + + try: + overall_score = max( + 0.0, min(100.0, float(judge_result.get("overall_score", 100))) + ) + except (TypeError, ValueError): + verbose_logger.warning( + "llm_as_a_judge: invalid overall_score from judge, failing open" + ) + return inputs + + passed = overall_score >= self.overall_threshold + + eval_info: "StandardLoggingEvalInformation" = { + "eval_name": self.guardrail_name or "", + "overall_score": overall_score, + "passed": passed, + "judge_model": self.judge_model, + "threshold": self.overall_threshold, + "verdicts": judge_result.get("verdicts", []), + } + _metadata = request_data.setdefault("metadata", {}) + existing = _metadata.get("eval_information") + if isinstance(existing, list): + existing.append(eval_info) + elif existing is not None: + _metadata["eval_information"] = [existing, eval_info] + else: + _metadata["eval_information"] = eval_info + + if not passed: + status = "guardrail_intervened" + if self.on_failure == "block": + raise HTTPException( + status_code=422, + detail={ + "error": "LLM judge rejected response: score below threshold", + "overall_score": overall_score, + "threshold": self.overall_threshold, + "verdicts": judge_result.get("verdicts", []), + }, + ) + + return inputs + + except HTTPException: + raise + except Exception as e: + verbose_logger.warning(f"llm_as_a_judge guardrail unexpected error: {e}") + return inputs + finally: + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_provider="llm_as_a_judge", + guardrail_json_response=judge_result, + request_data=request_data, + guardrail_status=status, + start_time=start_time.timestamp(), + end_time=datetime.now().timestamp(), + event_type=GuardrailEventHooks.post_call, + ) + + +def initialize_guardrail( + litellm_params: "LitellmParams", + guardrail: "Guardrail", +) -> LLMAsAJudgeGuardrail: + guardrail_name = guardrail.get("guardrail_name") + if not guardrail_name: + raise ValueError("llm_as_a_judge guardrail requires a guardrail_name") + + judge_model = _get_litellm_param(litellm_params, guardrail, "judge_model") + if not judge_model: + raise ValueError( + "llm_as_a_judge guardrail requires judge_model in litellm_params" + ) + + criteria = _get_litellm_param(litellm_params, guardrail, "criteria") or [] + if not criteria: + raise ValueError("llm_as_a_judge guardrail requires at least one criterion") + + weight_total = sum(float(c.get("weight", 0)) for c in criteria) + if abs(weight_total - 100) > 0.5: + raise ValueError( + f"llm_as_a_judge criterion weights must sum to 100 (got {weight_total})" + ) + + on_failure = _get_litellm_param(litellm_params, guardrail, "on_failure", "block") + if on_failure not in _VALID_ON_FAILURE: + raise ValueError( + f"llm_as_a_judge on_failure must be 'block' or 'log', got '{on_failure}'" + ) + + overall_threshold = float( + _get_litellm_param(litellm_params, guardrail, "overall_threshold", 80.0) + ) + + mode = _get_litellm_param(litellm_params, guardrail, "mode") + event_hook: Optional[GuardrailEventHooks] = None + if isinstance(mode, str) and mode in {e.value for e in GuardrailEventHooks}: + event_hook = GuardrailEventHooks(mode) + + instance = LLMAsAJudgeGuardrail( + guardrail_name=guardrail_name, + judge_model=judge_model, + criteria=criteria, + overall_threshold=overall_threshold, + on_failure=on_failure, + event_hook=event_hook, + default_on=bool( + _get_litellm_param(litellm_params, guardrail, "default_on", False) + ), + ) + litellm.logging_callback_manager.add_litellm_callback(instance) + return instance + + +__all__ = [ + "LLMAsAJudgeGuardrail", + "initialize_guardrail", +] diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 96175877d6..868b23756d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -34,6 +34,9 @@ from .guardrail_initializers import ( initialize_presidio, initialize_tool_permission, ) +from .guardrail_hooks.llm_as_a_judge import ( + initialize_guardrail as initialize_llm_as_a_judge, +) guardrail_initializer_registry = { SupportedGuardrailIntegrations.BEDROCK.value: initialize_bedrock, @@ -43,6 +46,7 @@ guardrail_initializer_registry = { SupportedGuardrailIntegrations.HIDE_SECRETS.value: initialize_hide_secrets, SupportedGuardrailIntegrations.TOOL_PERMISSION.value: initialize_tool_permission, SupportedGuardrailIntegrations.GRAYSWAN.value: initialize_grayswan, + SupportedGuardrailIntegrations.LLM_AS_A_JUDGE.value: initialize_llm_as_a_judge, } guardrail_class_registry: Dict[str, Type[CustomGuardrail]] = { diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index dbad25ffce..889c781004 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -107,6 +107,7 @@ def _get_spend_logs_metadata( model_map_information=None, usage_object=None, guardrail_information=None, + eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, attempted_retries=None, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 81cd6e5ecb..8eadb1e21e 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -91,6 +91,7 @@ class SupportedGuardrailIntegrations(Enum): BLOCK_CODE_EXECUTION = "block_code_execution" AKTO = "akto" MCP_JWT_SIGNER = "mcp_jwt_signer" + LLM_AS_A_JUDGE = "llm_as_a_judge" class Role(Enum): @@ -886,6 +887,8 @@ class ApplyGuardrailRequest(BaseModel): text: str language: Optional[str] = None entities: Optional[List[PiiEntityType]] = None + input_type: str = "request" + messages: Optional[List[Dict[str, Any]]] = None class ApplyGuardrailResponse(BaseModel): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c347956cba..a212d56c1a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2760,6 +2760,29 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): """Risk score 0-10 indicating how risky the request was (higher = riskier). Computed by the guardrail provider.""" +class EvalVerdict(TypedDict, total=False): + criterion_name: str + score: float # 0-100 + reasoning: str + passed: bool + weight: int # criterion weight (0-100) as configured in the guardrail + + +class StandardLoggingEvalInformation(TypedDict, total=False): + eval_id: Optional[str] + eval_name: str + overall_score: float + passed: bool + judge_model: str + iteration: int + eval_error: Optional[str] + start_time: str + end_time: str + duration: float + verdicts: List[Any] + threshold: Optional[float] + + class GuardrailTracingDetail(TypedDict, total=False): """ Typed fields for guardrail tracing metadata. diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 109061a10c..f4cc973517 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -42,6 +42,7 @@ ignored_keys = [ "metadata.cold_storage_object_key", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.eval_information", ] diff --git a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py new file mode 100644 index 0000000000..c9fde4ffba --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py @@ -0,0 +1,224 @@ +"""Unit tests for the LLM-as-a-Judge guardrail hook.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException + +from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import ( + LLMAsAJudgeGuardrail, + _build_judge_prompt, + _extract_text_from_content, + initialize_guardrail, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +CRITERIA_100 = [ + {"name": "Accuracy", "weight": 60, "description": "Is it accurate?"}, + {"name": "Safety", "weight": 40, "description": "Is it safe?"}, +] + + +def _make_guardrail(**overrides) -> LLMAsAJudgeGuardrail: + kwargs = dict( + guardrail_name="test_judge", + judge_model="gpt-4o-mini", + criteria=CRITERIA_100, + overall_threshold=80.0, + on_failure="block", + ) + kwargs.update(overrides) + return LLMAsAJudgeGuardrail(**kwargs) + + +def _make_verdict_response(overall_score: float) -> dict: + return { + "verdicts": [ + {"criterion_name": "Accuracy", "score": overall_score, "reasoning": "ok", "passed": True, "weight": 60}, + {"criterion_name": "Safety", "score": overall_score, "reasoning": "ok", "passed": True, "weight": 40}, + ], + "overall_score": overall_score, + } + + +# --------------------------------------------------------------------------- +# _extract_text_from_content +# --------------------------------------------------------------------------- + + +def test_extract_text_str(): + assert _extract_text_from_content("hello") == "hello" + + +def test_extract_text_multimodal_list(): + content = [{"type": "text", "text": "hello"}, {"type": "image_url", "url": "x"}] + assert _extract_text_from_content(content) == "hello" + + +def test_extract_text_unknown_type(): + assert _extract_text_from_content(42) == "" + + +# --------------------------------------------------------------------------- +# _build_judge_prompt +# --------------------------------------------------------------------------- + + +def test_build_judge_prompt_contains_criteria(): + prompt = _build_judge_prompt(CRITERIA_100, [], "response text") + assert "Accuracy" in prompt + assert "60%" in prompt + assert "Safety" in prompt + assert "response text" in prompt + + +def test_build_judge_prompt_missing_name_and_weight(): + criteria = [{"description": "check it"}] + prompt = _build_judge_prompt(criteria, [], "resp") + assert "0%" in prompt + + +# --------------------------------------------------------------------------- +# initialize_guardrail — validation +# --------------------------------------------------------------------------- + + +def _make_litellm_params(**overrides): + params = MagicMock() + for attr in ("guardrail_name", "judge_model", "criteria", "on_failure", "overall_threshold", "mode", "default_on"): + setattr(params, attr, None) + for k, v in overrides.items(): + setattr(params, k, v) + return params + + +def _make_guardrail_dict(name="g", **litellm_params_overrides): + raw = {"judge_model": "gpt-4o-mini", "criteria": CRITERIA_100, "on_failure": "block", "overall_threshold": 80.0} + raw.update(litellm_params_overrides) + return {"guardrail_name": name, "litellm_params": raw} + + +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.logging_callback_manager") +def test_initialize_guardrail_ok(mock_mgr): + lp = _make_litellm_params() + g = _make_guardrail_dict() + instance = initialize_guardrail(lp, g) + assert isinstance(instance, LLMAsAJudgeGuardrail) + mock_mgr.add_litellm_callback.assert_called_once_with(instance) + + +def test_initialize_guardrail_missing_judge_model(): + lp = _make_litellm_params() + g = _make_guardrail_dict(judge_model=None) + g["litellm_params"].pop("judge_model") + with pytest.raises(ValueError, match="judge_model"): + initialize_guardrail(lp, g) + + +def test_initialize_guardrail_weight_sum_not_100(): + lp = _make_litellm_params() + bad_criteria = [{"name": "A", "weight": 50, "description": "d"}] + g = _make_guardrail_dict(criteria=bad_criteria) + with pytest.raises(ValueError, match="100"): + initialize_guardrail(lp, g) + + +def test_initialize_guardrail_invalid_on_failure(): + lp = _make_litellm_params() + g = _make_guardrail_dict(on_failure="explode") + with pytest.raises(ValueError, match="on_failure"): + initialize_guardrail(lp, g) + + +# --------------------------------------------------------------------------- +# apply_guardrail — enforcement paths +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_apply_guardrail_pre_call_passthrough(): + guardrail = _make_guardrail() + inputs = {"texts": ["some text"]} + result = await guardrail.apply_guardrail(inputs, {}, "request") + assert result is inputs + + +@pytest.mark.asyncio +async def test_apply_guardrail_empty_response_passthrough(): + guardrail = _make_guardrail() + inputs = {"texts": []} + result = await guardrail.apply_guardrail(inputs, {}, "response") + assert result is inputs + + +@pytest.mark.asyncio +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion") +async def test_apply_guardrail_passes_above_threshold(mock_completion): + mock_completion.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(90.0))))] + ) + guardrail = _make_guardrail(overall_threshold=80.0) + inputs = {"texts": ["good response"]} + request_data: dict = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} + result = await guardrail.apply_guardrail(inputs, request_data, "response") + assert result is inputs + assert request_data["metadata"]["eval_information"]["passed"] is True + + +@pytest.mark.asyncio +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion") +async def test_apply_guardrail_blocks_below_threshold(mock_completion): + mock_completion.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(50.0))))] + ) + guardrail = _make_guardrail(overall_threshold=80.0, on_failure="block") + inputs = {"texts": ["bad response"]} + request_data: dict = {"messages": [], "metadata": {}} + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail(inputs, request_data, "response") + assert exc_info.value.status_code == 422 + + +@pytest.mark.asyncio +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion") +async def test_apply_guardrail_log_mode_does_not_block(mock_completion): + mock_completion.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content=json.dumps(_make_verdict_response(50.0))))] + ) + guardrail = _make_guardrail(overall_threshold=80.0, on_failure="log") + inputs = {"texts": ["bad response"]} + request_data: dict = {"messages": [], "metadata": {}} + result = await guardrail.apply_guardrail(inputs, request_data, "response") + assert result is inputs + assert request_data["metadata"]["eval_information"]["passed"] is False + + +@pytest.mark.asyncio +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion") +async def test_apply_guardrail_judge_error_fails_open(mock_completion): + mock_completion.side_effect = RuntimeError("judge down") + guardrail = _make_guardrail() + inputs = {"texts": ["response"]} + request_data: dict = {"messages": [], "metadata": {}} + result = await guardrail.apply_guardrail(inputs, request_data, "response") + assert result is inputs + + +@pytest.mark.asyncio +@patch("litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge.litellm.acompletion") +async def test_apply_guardrail_clamps_score(mock_completion): + response_payload = {"verdicts": [], "overall_score": 150} + mock_completion.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content=json.dumps(response_payload)))] + ) + guardrail = _make_guardrail(overall_threshold=80.0) + inputs = {"texts": ["response"]} + request_data: dict = {"messages": [], "metadata": {}} + result = await guardrail.apply_guardrail(inputs, request_data, "response") + assert result is inputs + assert request_data["metadata"]["eval_information"]["overall_score"] == 100.0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 2370d5df30..6f4e94d2c9 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -376,6 +376,7 @@ ignored_keys = [ "metadata.error_information", "metadata.attempted_retries", "metadata.max_retries", + "metadata.eval_information", ] MODEL_LIST = [ diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index 4cbe5664c6..c91a6e85cd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -1,7 +1,7 @@ import { Form, Input, Modal, Select, Tag, Typography, Button } from "antd"; import React, { useEffect, useMemo, useState } from "react"; import NotificationsManager from "../molecules/notifications_manager"; -import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings } from "../networking"; +import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings, modelAvailableCall } from "../networking"; import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { choiceToSkipSystemForCreate, @@ -11,10 +11,12 @@ import { populateGuardrailProviderMap, populateGuardrailProviders, shouldRenderContentFilterConfigSettings, + shouldRenderLLMJudgeFields, shouldRenderPIIConfigSettings, } from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; +import LLMJudgeFields from "./llm_judge/LLMJudgeFields"; import PiiConfiguration from "./pii_configuration"; import ToolPermissionRulesEditor, { ToolPermissionConfig } from "./tool_permission/ToolPermissionRulesEditor"; @@ -126,6 +128,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const [onViolation, setOnViolation] = useState<"warn" | "end_session">("warn"); const [realtimeViolationMessage, setRealtimeViolationMessage] = useState(""); const [endpointSettingsOpen, setEndpointSettingsOpen] = useState(false); + const [availableModels, setAvailableModels] = useState([]); const [toolPermissionConfig, setToolPermissionConfig] = useState({ rules: [], @@ -149,13 +152,17 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const fetchData = async () => { try { // Parallel requests for speed - const [uiSettings, providerParamsResp] = await Promise.all([ + const [uiSettings, providerParamsResp, modelsResp] = await Promise.all([ getGuardrailUISettings(accessToken), getGuardrailProviderSpecificParams(accessToken), + modelAvailableCall(accessToken, "", "").catch(() => null), ]); setGuardrailSettings(uiSettings); setProviderParams(providerParamsResp); + if (modelsResp?.data) { + setAvailableModels(modelsResp.data.map((m: any) => m.id)); + } // Populate dynamic providers from API response populateGuardrailProviders(providerParamsResp); @@ -242,6 +249,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a on_disallowed_action: "block", violation_message_template: "", }); + + // Default LLM-as-a-Judge to post_call mode + if (value === "LlmAsAJudge") { + form.setFieldsValue({ mode: "post_call" }); + } }; const handleEntitySelect = (entity: string) => { @@ -510,6 +522,29 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a } } + if (guardrailProvider === "llm_as_a_judge") { + const criteria: any[] = values.criteria || []; + if (criteria.length === 0) { + NotificationsManager.fromBackend("Add at least one evaluation criterion"); + setLoading(false); + return; + } + const weightTotal = criteria.reduce((sum: number, c: any) => sum + (Number(c?.weight) || 0), 0); + if (weightTotal !== 100) { + NotificationsManager.fromBackend(`Criterion weights must sum to 100% (currently ${weightTotal}%)`); + setLoading(false); + return; + } + guardrailData.litellm_params.judge_model = values.judge_model; + guardrailData.litellm_params.overall_threshold = values.overall_threshold ?? 80; + guardrailData.litellm_params.on_failure = values.on_failure ?? "block"; + guardrailData.litellm_params.criteria = criteria.map((c: any) => ({ + name: c.name, + weight: Number(c.weight), + description: c.description || "", + })); + } + if (guardrailProvider === "tool_permission") { if (toolPermissionConfig.rules.length === 0) { NotificationsManager.fromBackend("Add at least one tool permission rule"); @@ -549,7 +584,8 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a console.log("values: ", JSON.stringify(values)); // Use pre-fetched provider params to copy recognised params - if (providerParams && selectedProvider) { + // Skip for providers that handle their own litellm_params (llm_as_a_judge, tool_permission, content filter, PII) + if (providerParams && selectedProvider && guardrailProvider !== "llm_as_a_judge") { const providerKey = guardrail_provider_map[selectedProvider]?.toLowerCase(); console.log("providerKey: ", providerKey); const providerSpecificParams = providerParams[providerKey] || {}; @@ -769,7 +805,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a {/* Use the GuardrailProviderFields component to render provider-specific fields */} - {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && ( + {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && ( = ({ visible, onClose, a if (shouldRenderContentFilterConfigSettings(selectedProvider)) { return renderContentFilterConfiguration("categories"); } + if (shouldRenderLLMJudgeFields(selectedProvider)) { + return ; + } return renderOptionalParams(); case 2: if (shouldRenderContentFilterConfigSettings(selectedProvider)) { diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index 8426a54008..5a1e93021a 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -16,6 +16,7 @@ export const populateGuardrailProviders = (providerParamsResponse: Record { @@ -49,6 +50,7 @@ export const guardrail_provider_map: Record = { ToolPermission: "tool_permission", BlockCodeExecution: "block_code_execution", Promptguard: "promptguard", + LlmAsAJudge: "llm_as_a_judge", }; // Function to populate provider map from API response - updates the original map @@ -103,6 +105,11 @@ export const shouldRenderContentFilterConfigSettings = (provider: string | null) return providerEnum === "LiteLLM Content Filter"; }; +export const shouldRenderLLMJudgeFields = (provider: string | null) => { + if (!provider) return false; + return guardrail_provider_map[provider] === "llm_as_a_judge"; +}; + const asset_logos_folder = "../ui/assets/logos/"; export const guardrailLogoMap: Record = { @@ -127,6 +134,7 @@ export const guardrailLogoMap: Record = { "Prompt Security": `${asset_logos_folder}prompt_security.png`, PromptGuard: `${asset_logos_folder}promptguard.svg`, "LiteLLM Content Filter": `${asset_logos_folder}litellm_logo.jpg`, + "LiteLLM LLM as a Judge": `${asset_logos_folder}litellm_logo.jpg`, "Akto": `${asset_logos_folder}akto.svg`, }; diff --git a/ui/litellm-dashboard/src/components/guardrails/llm_judge/LLMJudgeFields.tsx b/ui/litellm-dashboard/src/components/guardrails/llm_judge/LLMJudgeFields.tsx new file mode 100644 index 0000000000..73379d168a --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/llm_judge/LLMJudgeFields.tsx @@ -0,0 +1,191 @@ +"use client"; + +import React from "react"; +import { Form, Select, InputNumber, Input, Tooltip } from "antd"; +import { PlusOutlined, QuestionCircleOutlined } from "@ant-design/icons"; +import { Button } from "antd"; + +interface LLMJudgeFieldsProps { + availableModels: string[]; + form: any; +} + +const LLMJudgeFields: React.FC = ({ availableModels, form }) => { + return ( + <> +
+ After each LLM response, the Judge Model scores it 0–100 against your criteria. + If the weighted average falls below the threshold, the response is blocked (or logged). +
+ + + Judge Model  + + + + + } + rules={[{ required: true, message: "Select a judge model" }]} + > + + Block (return 422) + Log only + + + + + Evaluation Criteria  + + + + + } + > + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( +
+
+ + + + + + Weight + + + } + rules={[{ required: true, message: "Enter weight" }]} + style={{ flex: 1, marginBottom: 8 }} + > + + +
+ +
+
+ + + +
+ ))} + + {fields.length > 0 && ( + + {() => { + const allCriteria: any[] = form.getFieldValue("criteria") || []; + const weightTotal = allCriteria.reduce( + (sum: number, c: any) => sum + (Number(c?.weight) || 0), + 0, + ); + const weightOk = weightTotal === 100; + return ( +
+ Weights total: {weightTotal}%{weightOk ? " ✓" : " — must add up to 100%"} +
+ ); + }} +
+ )} + + )} +
+
+ + ); +}; + +export default LLMJudgeFields; diff --git a/ui/litellm-dashboard/src/components/view_logs/EvalViewer/EvalViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/EvalViewer/EvalViewer.tsx new file mode 100644 index 0000000000..e12c2917a0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/EvalViewer/EvalViewer.tsx @@ -0,0 +1,203 @@ +import React from "react"; +import { Card, Tag, Table, Typography, Space, Tooltip } from "antd"; +import { CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined } from "@ant-design/icons"; + +const { Text } = Typography; + +interface EvalVerdict { + criterion_name: string; + score: number; + reasoning: string; + passed: boolean; + weight?: number; +} + +interface EvalInformation { + eval_id?: string; + eval_name: string; + overall_score: number; + passed: boolean; + judge_model: string; + iteration?: number; + eval_error?: string | null; + verdicts?: EvalVerdict[]; + threshold?: number; +} + +interface EvalViewerProps { + data: EvalInformation | EvalInformation[]; +} + +export default function EvalViewer({ data }: EvalViewerProps) { + const entries: EvalInformation[] = Array.isArray(data) ? data : [data]; + + if (!entries.length) return null; + + return ( +
+
+ + + LLM Judge Results + +
+ + {entries.map((entry, idx) => ( + + ))} +
+ ); +} + +function EvalEntryCard({ entry }: { entry: EvalInformation }) { + const passed = entry.passed; + const scoreColor = passed ? "#52c41a" : "#ff4d4f"; + + // Filter out synthetic "Overall" row the judge sometimes appends — it's already in the header + const verdicts = (entry.verdicts || []).filter( + (v) => (v.criterion_name || "").toLowerCase() !== "overall" + ); + + const columns = [ + { + title: "Criterion", + dataIndex: "criterion_name", + key: "criterion_name", + width: 160, + render: (v: string) => {v}, + }, + { + title: "Weight", + dataIndex: "weight", + key: "weight", + width: 65, + render: (v: number) => + v != null ? ( + {v}% + ) : null, + }, + { + title: "Score", + dataIndex: "score", + key: "score", + width: 65, + render: (v: number) => ( + = 70 ? "#52c41a" : v >= 50 ? "#faad14" : "#ff4d4f", fontWeight: 600 }}> + {v} + + ), + }, + { + title: ( + + Weighted + + ), + key: "weighted", + width: 75, + render: (_: unknown, row: EvalVerdict) => { + if (row.weight == null) return null; + const contrib = (row.score * row.weight) / 100; + return ( + + {contrib % 1 === 0 ? contrib : contrib.toFixed(1)} + + ); + }, + }, + { + title: "Comment", + dataIndex: "reasoning", + key: "reasoning", + ellipsis: { showTitle: false }, + render: (v: string) => ( + + {v} + + ), + }, + ]; + + return ( + + {passed ? ( + + ) : ( + + )} + {entry.eval_name} + {passed ? "PASSED" : "FAILED"} + + + {entry.overall_score?.toFixed(0)} / 100 + {entry.threshold != null && ` (threshold: ${entry.threshold})`} + + + + } + extra={ + + {entry.judge_model && ( + + Judge: {entry.judge_model} + + )} + {entry.iteration != null && ( + + Iter: {entry.iteration + 1} + + )} + + } + > + {entry.eval_error && ( + + Judge error: {entry.eval_error} + + )} + + {verdicts.length > 0 ? ( + { + const hasWeights = verdicts.some((v) => v.weight != null); + if (!hasWeights) return null; + const total = verdicts.reduce( + (sum, v) => sum + (v.weight != null ? (v.score * v.weight) / 100 : 0), + 0 + ); + return ( + + + Total + + + + + + {total % 1 === 0 ? total : total.toFixed(1)} + + + + + ); + }} + /> + ) : ( + + Score: {entry.overall_score?.toFixed(1)} — no per-criterion breakdown available. + + )} + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 5608e3677f..cfa192db5a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -733,15 +733,15 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) )} - {/* ── Body: two columns ──────────────────────────────────── */} -
- {/* Left column: Request Lifecycle */} -
+ {/* ── Body: stacked ──────────────────────────────────────── */} +
+ {/* Request Lifecycle */} +
- {/* Right column: Evaluation Details */} -
+ {/* Evaluation Details */} +

Evaluation Details

diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 62b7307471..c0c6718693 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -4,6 +4,7 @@ import moment from "moment"; import { LogEntry } from "../columns"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import GuardrailViewer from "../GuardrailViewer/GuardrailViewer"; +import EvalViewer from "../EvalViewer/EvalViewer"; import { CostBreakdownViewer } from "../CostBreakdownViewer"; import { ConfigInfoMessage } from "../ConfigInfoMessage"; import { VectorStoreViewer } from "../VectorStoreViewer"; @@ -68,6 +69,10 @@ export function LogDetailContent({ logEntry, onOpenSettings, isLoadingDetails = const totalMaskedEntities = calculateTotalMaskedEntities(guardrailEntries); const primaryGuardrailLabel = getGuardrailLabel(guardrailEntries); + // LLM Judge data + const evalInfo = metadata?.eval_information; + const hasEvalData = evalInfo != null; + // Vector store data const hasVectorStoreData = checkHasVectorStoreData(metadata); @@ -190,6 +195,9 @@ export function LogDetailContent({ logEntry, onOpenSettings, isLoadingDetails =
)} + {/* LLM Judge Results */} + {hasEvalData && } + {/* Vector Store Data */} {hasVectorStoreData && }