From 1a5133f2290ae7d5ef566cb09dcdc6ceed8d5a55 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 18 Feb 2026 21:39:19 -0800 Subject: [PATCH] Competitor guardrails: streaming discovery, variations, pre/post split (#21533) * fix aviation safety topic filter: remove overly broad exceptions, add cockpit access block words * fix airline brand protection filter: add identifier words, competitor/ops block words, tighten exceptions * update policy templates with competitor pre/post guardrails and streaming enrichment * sync policy_templates_backup.json with policy_templates.json * add streaming enrichment endpoint, competitor variations, and model selection * add streaming enrichPolicyTemplate networking function * rewrite template parameter modal with streaming tags, AI/manual toggle, model selector * update policies panel to pass enrichment options and show discovered competitors * show discovered competitors as tags in guardrail selection modal * use llm_router instead of litellm.acompletion, extract helpers, move constants * validate competitors list size, cap variation prompt input * add refinement instruction support for competitor discovery * add instruction and existingCompetitors params to streaming enrichment * add refinement input for iterating on competitor list with AI * emit status events during variation generation so UI shows progress * add onStatus callback to streaming enrichment * show status spinner during variation generation, widen modal to 700px * add tests for competitor enrichment helper functions --- litellm/constants.py | 5 + litellm/policy_templates_backup.json | 152 +++++- .../airline_brand_protection.yaml | 34 +- .../aviation_safety_topics.yaml | 8 +- .../management_endpoints/policy_endpoints.py | 448 +++++++++++++++--- policy_templates.json | 152 +++++- .../test_policy_endpoints.py | 253 ++++++++++ .../src/components/networking.tsx | 81 +++- .../policies/guardrail_selection_modal.tsx | 25 + .../src/components/policies/index.tsx | 20 +- .../policies/template_parameter_modal.tsx | 345 ++++++++++++-- 11 files changed, 1383 insertions(+), 140 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 17ad742e41..e4d3fa39e9 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1474,3 +1474,8 @@ MICROSOFT_USER_FIRST_NAME_ATTRIBUTE = str( MICROSOFT_USER_LAST_NAME_ATTRIBUTE = str( os.getenv("MICROSOFT_USER_LAST_NAME_ATTRIBUTE", "surname") ) + +# Policy template enrichment +MAX_COMPETITOR_NAMES = 100 +COMPETITOR_LLM_TEMPERATURE = 0.3 +DEFAULT_COMPETITOR_DISCOVERY_MODEL = "gpt-4o-mini" diff --git a/litellm/policy_templates_backup.json b/litellm/policy_templates_backup.json index f27f20187f..0dfa070aaa 100644 --- a/litellm/policy_templates_backup.json +++ b/litellm/policy_templates_backup.json @@ -1522,7 +1522,13 @@ "guardrails": [ "aviation-ops-data-protection", "aviation-safety-topic-filter", - "airline-brand-protection-filter" + "airline-brand-protection-filter", + "competitor-name-input-blocker", + "competitor-name-output-blocker", + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "complexity": "High", "parameters": [ @@ -1531,9 +1537,14 @@ "label": "Your Airline / Brand Name", "type": "text", "required": true, - "placeholder": "e.g. Emirates" + "placeholder": "e.g. Acme Airlines" } ], + "llm_enrichment": { + "parameter": "brand_name", + "prompt": "List the top 30 direct competitors of {{brand_name}} in the airline industry. Include major international carriers, regional competitors, and low-cost carriers that operate on overlapping routes. Return ONLY airline/brand names, one per line, no numbering, no explanations.", + "result_key": "competitors" + }, "guardrailDefinitions": [ { "guardrail_name": "aviation-ops-data-protection", @@ -1675,6 +1686,72 @@ "guardrail_info": { "description": "Blocks AI-generated fake incident reports, unauthorized statements, and reputation-damaging content about your brand (runs on output)" } + }, + { + "guardrail_name": "competitor-name-input-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs that mention competitor names (pre_call)" + } + }, + { + "guardrail_name": "competitor-name-output-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks AI outputs that mention competitor names (post_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks user requests asking to recommend competitors (pre_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-output-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks AI from recommending or suggesting competitor services (post_call)" + } + }, + { + "guardrail_name": "competitor-comparison-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs requesting unfavorable brand comparisons (pre_call)" + } + }, + { + "guardrail_name": "competitor-comparison-output-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks AI outputs with unfavorable brand comparisons (post_call)" + } } ], "templateData": { @@ -1683,7 +1760,13 @@ "guardrails_add": [ "aviation-ops-data-protection", "aviation-safety-topic-filter", - "airline-brand-protection-filter" + "airline-brand-protection-filter", + "competitor-name-input-blocker", + "competitor-name-output-blocker", + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "guardrails_remove": [] }, @@ -1812,9 +1895,12 @@ "iconColor": "text-orange-500", "iconBg": "bg-orange-50", "guardrails": [ + "competitor-input-blocker", "competitor-output-blocker", - "competitor-recommendation-filter", - "competitor-comparison-filter" + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "complexity": "Medium", "parameters": [ @@ -1823,15 +1909,26 @@ "label": "Your Brand Name", "type": "text", "required": true, - "placeholder": "e.g. Emirates" + "placeholder": "e.g. Acme Airlines" } ], "llm_enrichment": { "parameter": "brand_name", - "prompt": "List the top 10 direct competitors of {{brand_name}} in the same industry. Return ONLY company/brand names, one per line, no numbering, no explanations.", + "prompt": "List the top 30 direct competitors of {{brand_name}} in the same industry. Return ONLY company/brand names, one per line, no numbering, no explanations.", "result_key": "competitors" }, "guardrailDefinitions": [ + { + "guardrail_name": "competitor-input-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs that mention competitor brands (pre_call)" + } + }, { "guardrail_name": "competitor-output-blocker", "litellm_params": { @@ -1840,39 +1937,64 @@ "blocked_words": "{{competitors_blocked_words}}" }, "guardrail_info": { - "description": "Blocks AI outputs that mention or promote competitor brands (auto-discovered via LLM)" + "description": "Blocks AI outputs that mention competitor brands (post_call)" } }, { - "guardrail_name": "competitor-recommendation-filter", + "guardrail_name": "competitor-recommendation-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks user requests asking to recommend competitors (pre_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-output-filter", "litellm_params": { "guardrail": "litellm_content_filter", "mode": "post_call", "blocked_words": "{{competitor_recommendation_words}}" }, "guardrail_info": { - "description": "Blocks AI from recommending, suggesting, or directing users to competitor services" + "description": "Blocks AI from recommending or suggesting competitor services (post_call)" } }, { - "guardrail_name": "competitor-comparison-filter", + "guardrail_name": "competitor-comparison-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs requesting unfavorable brand comparisons (pre_call)" + } + }, + { + "guardrail_name": "competitor-comparison-output-filter", "litellm_params": { "guardrail": "litellm_content_filter", "mode": "post_call", "blocked_words": "{{competitor_comparison_words}}" }, "guardrail_info": { - "description": "Blocks unfavorable comparisons between your brand and competitors in AI outputs" + "description": "Blocks AI outputs with unfavorable brand comparisons (post_call)" } } ], "templateData": { "policy_name": "competitor-mention-detection", - "description": "Detects and blocks competitor mentions in AI outputs. Uses LLM-powered competitor discovery based on your brand name.", + "description": "Detects and blocks competitor mentions in both inputs and outputs. Uses LLM-powered competitor discovery based on your brand name.", "guardrails_add": [ + "competitor-input-blocker", "competitor-output-blocker", - "competitor-recommendation-filter", - "competitor-comparison-filter" + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "guardrails_remove": [] }, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/airline_brand_protection.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/airline_brand_protection.yaml index e2b7d13861..385fad87bc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/airline_brand_protection.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/airline_brand_protection.yaml @@ -11,7 +11,11 @@ identifier_words: - "airline" - "carrier" - "airways" + - "flight" - "flights" + - "Emirates" + - "Etihad" + - "Qatar Airways" additional_block_words: # Fake incident reports @@ -57,6 +61,25 @@ additional_block_words: - "unsafe" - "dangerous" - "negligence" + # Competitor recommendations + - "consider etihad" + - "try qatar" + - "switch to" + - "better value" + - "better product" + - "best product" + - "caught up" + - "competitors have" + - "recommend qatar" + - "recommend etihad" + - "consider switching" + # Ops data leakage + - "crew roster" + - "crew schedule" + - "crew list" + - "pilot names" + - "gate assignment" + - "cabin crew" always_block_keywords: - keyword: "plane crash" @@ -73,13 +96,20 @@ always_block_keywords: severity: "high" - keyword: "fleet grounded" severity: "high" + - keyword: "crew roster" + severity: "high" + - keyword: "crew schedule" + severity: "high" + - keyword: "pilot names" + severity: "high" + - keyword: "gate assignment" + severity: "high" exceptions: - "how safe is" - "safety record" - "safety rating" - - "what is" - - "explain" + - "what is the baggage" - "historical" - "aviation history" - "customer review" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/aviation_safety_topics.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/aviation_safety_topics.yaml index 4c97739874..2255c1a6b5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/aviation_safety_topics.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/aviation_safety_topics.yaml @@ -51,6 +51,11 @@ additional_block_words: - "break into cockpit" - "cockpit intrusion" - "cockpit breach" + - "locking mechanism" + - "open the cockpit" + - "cockpit door open" + - "enter the cockpit" + - "access the cockpit" # Aircraft vulnerabilities - "vulnerability" - "exploit" @@ -115,6 +120,3 @@ exceptions: - "how to report" - "incident report" - "safety culture" - - "what is" - - "explain" - - "research on" diff --git a/litellm/proxy/management_endpoints/policy_endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints.py index 8eec697462..7bc0b87a0d 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints.py @@ -9,14 +9,29 @@ All /policy management endpoints /policy/templates - Get policy templates (GitHub with local fallback) """ +import copy import json import os -from typing import TYPE_CHECKING, List, Literal, Optional, TypedDict, cast +from typing import ( + TYPE_CHECKING, + AsyncIterator, + List, + Literal, + Optional, + TypedDict, + cast, +) from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + COMPETITOR_LLM_TEMPERATURE, + DEFAULT_COMPETITOR_DISCOVERY_MODEL, + MAX_COMPETITOR_NAMES, +) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -511,6 +526,48 @@ async def get_policy_templates( class EnrichTemplateRequest(BaseModel): template_id: str parameters: dict + model: Optional[str] = None + competitors: Optional[List[str]] = Field( + default=None, + max_length=MAX_COMPETITOR_NAMES, + description="Optional list of competitor names", + ) + instruction: Optional[str] = Field( + default=None, + description="Refinement instruction for modifying the competitor list (e.g. 'add 10 more from Asia')", + ) + + +def _validate_enrichment_request(data: EnrichTemplateRequest) -> tuple[dict, dict, str]: + """ + Validate enrichment request and return (template, llm_enrichment, brand_name). + + Raises HTTPException on validation failure. + """ + templates = _load_policy_templates_from_local_backup() + template = next((t for t in templates if t.get("id") == data.template_id), None) + if template is None: + raise HTTPException(status_code=404, detail=f"Template '{data.template_id}' not found") + + llm_enrichment = template.get("llm_enrichment") + if llm_enrichment is None: + raise HTTPException(status_code=400, detail="Template does not support LLM enrichment") + + # Validate competitors list size if provided + if data.competitors and len(data.competitors) > MAX_COMPETITOR_NAMES: + raise HTTPException( + status_code=400, + detail=f"competitors list exceeds maximum of {MAX_COMPETITOR_NAMES}", + ) + + brand_name = data.parameters.get(llm_enrichment["parameter"], "") + if not brand_name: + raise HTTPException( + status_code=400, + detail=f"Parameter '{llm_enrichment['parameter']}' is required", + ) + + return template, llm_enrichment, brand_name @router.post( @@ -530,108 +587,311 @@ async def enrich_policy_template( Calls an onboarded LLM to discover competitors for the given brand name, then returns enriched guardrailDefinitions with the discovered data populated. """ - templates = _load_policy_templates_from_local_backup() - template = next((t for t in templates if t.get("id") == data.template_id), None) - if template is None: - raise HTTPException(status_code=404, detail=f"Template '{data.template_id}' not found") + template, llm_enrichment, brand_name = _validate_enrichment_request(data) + model = data.model or DEFAULT_COMPETITOR_DISCOVERY_MODEL - llm_enrichment = template.get("llm_enrichment") - if llm_enrichment is None: - raise HTTPException( - status_code=400, - detail="Template does not support LLM enrichment", + if data.competitors: + competitors = data.competitors + else: + prompt = llm_enrichment["prompt"].replace( + "{{" + llm_enrichment["parameter"] + "}}", brand_name ) + competitors = await _discover_competitors_via_llm(prompt, model=model) - brand_name = data.parameters.get(llm_enrichment["parameter"], "") - if not brand_name: - raise HTTPException( - status_code=400, - detail=f"Parameter '{llm_enrichment['parameter']}' is required", - ) - - prompt = llm_enrichment["prompt"].replace( - "{{" + llm_enrichment["parameter"] + "}}", brand_name - ) - - competitors = await _discover_competitors_via_llm(prompt) + variations_map = await _generate_competitor_variations(competitors, model=model) enriched_definitions = _build_competitor_guardrail_definitions( template.get("guardrailDefinitions", []), competitors, brand_name, + variations_map, ) - return {"guardrailDefinitions": enriched_definitions, "competitors": competitors} + return { + "guardrailDefinitions": enriched_definitions, + "competitors": competitors, + "competitor_variations": variations_map, + } -async def _discover_competitors_via_llm(prompt: str) -> list: - """Call an onboarded LLM to discover competitor names.""" - import litellm +def _build_refinement_prompt( + instruction: str, + existing_competitors: list[str], + brand_name: str, +) -> str: + """Build a prompt for refining the competitor list based on user instruction.""" + existing_list = ", ".join(existing_competitors) + return ( + f"I have a brand called '{brand_name}' and the following competitor list:\n" + f"{existing_list}\n\n" + f"User instruction: {instruction}\n\n" + "Return ONLY the NEW names to add (not the existing ones), one per line, " + "no numbering, no explanations. If the instruction asks to remove names, " + "return nothing." + ) + + +async def _stream_llm_competitor_names( + prompt: str, + model: str, + existing: list[str], +) -> AsyncIterator[tuple[Optional[str], bool]]: + """ + Stream competitor names from LLM. Yields (name, is_error) tuples. + + Deduplicates against existing names (case-insensitive). + """ + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + raise ValueError("LLM router not initialized") + + existing_lower = {n.lower() for n in existing} + response = await llm_router.acompletion( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=COMPETITOR_LLM_TEMPERATURE, + stream=True, + ) + buffer = "" + count = len(existing) + async for chunk in response: # type: ignore[union-attr] + delta = chunk.choices[0].delta.content or "" + buffer += delta + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + name = _clean_competitor_line(line) + if name and name.lower() not in existing_lower and count < MAX_COMPETITOR_NAMES: + existing_lower.add(name.lower()) + count += 1 + yield name, False + # Handle remaining buffer + name = _clean_competitor_line(buffer) + if name and name.lower() not in existing_lower and count < MAX_COMPETITOR_NAMES: + yield name, False + + +async def _stream_competitor_events( + data: EnrichTemplateRequest, + template: dict, + llm_enrichment: dict, + brand_name: str, + model: str, +) -> AsyncIterator[str]: + """Stream competitor names as SSE events, then emit a final 'done' event.""" + competitors: list[str] = list(data.competitors or []) + + if data.instruction and competitors: + # Refinement mode: keep existing, stream only new names + for comp in competitors: + yield f"data: {json.dumps({'type': 'competitor', 'name': comp})}\n\n" + + refinement_prompt = _build_refinement_prompt( + data.instruction, competitors, brand_name + ) + try: + async for name, _ in _stream_llm_competitor_names( + refinement_prompt, model, competitors + ): + if name: + competitors.append(name) + yield f"data: {json.dumps({'type': 'competitor', 'name': name})}\n\n" + except Exception as e: + verbose_proxy_logger.error("LLM competitor refinement failed: %s", e) + yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n" + return + elif data.competitors and not data.instruction: + # Free-form mode (no instruction): just emit existing + for comp in competitors: + yield f"data: {json.dumps({'type': 'competitor', 'name': comp})}\n\n" + else: + # Initial discovery mode + prompt = llm_enrichment["prompt"].replace( + "{{" + llm_enrichment["parameter"] + "}}", brand_name + ) + try: + async for name, _ in _stream_llm_competitor_names( + prompt, model, [] + ): + if name: + competitors.append(name) + yield f"data: {json.dumps({'type': 'competitor', 'name': name})}\n\n" + except Exception as e: + verbose_proxy_logger.error("LLM competitor streaming failed: %s", e) + yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n" + return + + yield f"data: {json.dumps({'type': 'status', 'message': f'Generating alternate spellings for {len(competitors)} competitors...'})}\n\n" + variations_map = await _generate_competitor_variations(competitors, model=model) + + total_variations = sum(len(v) for v in variations_map.values()) + yield f"data: {json.dumps({'type': 'status', 'message': f'Building guardrail definitions with {total_variations} variations...'})}\n\n" + enriched_definitions = _build_competitor_guardrail_definitions( + template.get("guardrailDefinitions", []), + competitors, + brand_name, + variations_map, + ) + + yield f"data: {json.dumps({'type': 'done', 'competitors': competitors, 'competitor_variations': variations_map, 'guardrailDefinitions': enriched_definitions})}\n\n" + + +@router.post( + "/policy/templates/enrich/stream", + tags=["policy management"], + dependencies=[Depends(user_api_key_auth)], +) +async def enrich_policy_template_stream( + data: EnrichTemplateRequest, + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Stream competitor names as SSE events as the LLM generates them. + + Events: + - data: {"type": "competitor", "name": "..."} — each competitor as discovered + - data: {"type": "done", "competitors": [...], "competitor_variations": {...}, "guardrailDefinitions": [...]} + """ + template, llm_enrichment, brand_name = _validate_enrichment_request(data) + model = data.model or DEFAULT_COMPETITOR_DISCOVERY_MODEL + + return StreamingResponse( + _stream_competitor_events(data, template, llm_enrichment, brand_name, model), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +def _clean_competitor_line(line: str) -> Optional[str]: + """Strip numbering, bullets, and whitespace from a competitor name line.""" + name = line.strip().strip(".-) ").strip() + return name if name and len(name) > 1 else None + + +async def _generate_competitor_variations( + competitors: list, model: str = DEFAULT_COMPETITOR_DISCOVERY_MODEL +) -> dict: + """Generate common misspellings, abbreviations, and alternate names for each competitor.""" + if not competitors: + return {} + + # Cap the list to prevent oversized prompts + capped = competitors[:MAX_COMPETITOR_NAMES] + names_list = "\n".join(capped) + prompt = ( + "For each company/brand name below, list 3-5 common misspellings, abbreviations, " + "and alternate names that people might type. Include typos, missing spaces, " + "wrong suffixes (e.g. 'Airlines' vs 'Airways' vs 'Airline'), and common shortcuts.\n\n" + f"Names:\n{names_list}\n\n" + "Return the result as one line per variation in the format:\n" + "OriginalName: variation1, variation2, variation3\n" + "Use the EXACT original name before the colon. No numbering, no extra text." + ) try: - response = await litellm.acompletion( - model="gpt-4o-mini", + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + raise ValueError("LLM router not initialized") + response = await llm_router.acompletion( + model=model, messages=[{"role": "user", "content": prompt}], - temperature=0.3, + temperature=COMPETITOR_LLM_TEMPERATURE, + ) + raw = response.choices[0].message.content or "" # type: ignore + return _parse_variations_response(raw, capped) + except Exception as e: + verbose_proxy_logger.error("LLM competitor variation generation failed: %s", e) + return {} + + +def _parse_variations_response(raw: str, competitors: list) -> dict[str, list[str]]: + """Parse the LLM response for competitor variations into a name -> variations map.""" + # Build a lowercase lookup for case-insensitive matching + lower_to_canonical = {comp.lower(): comp for comp in competitors} + variations_map: dict[str, list[str]] = {} + + for line in raw.strip().split("\n"): + if ":" not in line: + continue + name, _, variations_str = line.partition(":") + canonical = lower_to_canonical.get(name.strip().lower()) + if canonical is None: + continue + variations = [ + v.strip() + for v in variations_str.split(",") + if v.strip() and v.strip().lower() != canonical.lower() + ] + variations_map[canonical] = variations + + return variations_map + + +async def _discover_competitors_via_llm( + prompt: str, model: str = DEFAULT_COMPETITOR_DISCOVERY_MODEL +) -> list: + """Call an onboarded LLM to discover competitor names.""" + try: + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + raise ValueError("LLM router not initialized") + response = await llm_router.acompletion( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=COMPETITOR_LLM_TEMPERATURE, ) raw = response.choices[0].message.content or "" # type: ignore competitors = [ - line.strip().strip(".-) ").strip() + name for line in raw.strip().split("\n") - if line.strip() and len(line.strip()) > 1 + if (name := _clean_competitor_line(line)) is not None ] - return competitors[:15] + return competitors[:MAX_COMPETITOR_NAMES] except Exception as e: verbose_proxy_logger.error("LLM competitor discovery failed: %s", e) return [] +def _build_all_names_per_competitor( + competitors: list[str], variations_map: dict[str, list[str]] +) -> dict[str, list[str]]: + """Build canonical + variation name lists for each competitor.""" + return { + comp: [comp] + variations_map.get(comp, []) + for comp in competitors + } + + def _build_competitor_guardrail_definitions( definitions: list, competitors: list, brand_name: str, + variations_map: Optional[dict] = None, ) -> list: - """Build enriched guardrailDefinitions with competitor names populated.""" - import copy - + """Build enriched guardrailDefinitions with competitor names and variations populated.""" + variations_map = variations_map or {} enriched = copy.deepcopy(definitions) + all_names = _build_all_names_per_competitor(competitors, variations_map) - output_blocked = [ - {"keyword": comp, "action": "BLOCK", "description": f"Competitor: {comp}"} - for comp in competitors - ] - - recommendation_blocked = [] - for comp in competitors: - recommendation_blocked.append( - {"keyword": f"try {comp}", "action": "BLOCK", "description": "Recommendation to competitor"} - ) - recommendation_blocked.append( - {"keyword": f"use {comp}", "action": "BLOCK", "description": "Recommendation to competitor"} - ) - recommendation_blocked.append( - {"keyword": f"switch to {comp}", "action": "BLOCK", "description": "Recommendation to competitor"} - ) - recommendation_blocked.append( - {"keyword": f"consider {comp}", "action": "BLOCK", "description": "Recommendation to competitor"} - ) - - comparison_blocked = [] - for comp in competitors: - comparison_blocked.append( - {"keyword": f"{comp} is better", "action": "BLOCK", "description": "Unfavorable comparison"} - ) - comparison_blocked.append( - {"keyword": f"better than {brand_name}", "action": "BLOCK", "description": "Unfavorable comparison"} - ) - comparison_blocked.append( - {"keyword": f"{brand_name} is worse", "action": "BLOCK", "description": "Unfavorable comparison"} - ) + output_blocked = _build_name_blocked_words(competitors, all_names) + recommendation_blocked = _build_recommendation_blocked_words(competitors, all_names) + comparison_blocked = _build_comparison_blocked_words(competitors, all_names, brand_name) blocked_words_map = { "competitor-output-blocker": output_blocked, + "competitor-input-blocker": output_blocked, + "competitor-name-blocker": output_blocked, + "competitor-name-input-blocker": output_blocked, + "competitor-name-output-blocker": output_blocked, "competitor-recommendation-filter": recommendation_blocked, + "competitor-recommendation-input-filter": recommendation_blocked, + "competitor-recommendation-output-filter": recommendation_blocked, "competitor-comparison-filter": comparison_blocked, + "competitor-comparison-input-filter": comparison_blocked, + "competitor-comparison-output-filter": comparison_blocked, } for defn in enriched: @@ -640,3 +900,59 @@ def _build_competitor_guardrail_definitions( defn["litellm_params"]["blocked_words"] = blocked_words_map[guardrail_name] return enriched + + +def _build_name_blocked_words( + competitors: list[str], all_names: dict[str, list[str]] +) -> list[dict]: + """Build blocked word entries for direct competitor name mentions.""" + result = [] + for comp in competitors: + for name in all_names[comp]: + desc = f"Competitor: {comp}" if name == comp else f"Competitor variation ({comp}): {name}" + result.append({"keyword": name, "action": "BLOCK", "description": desc}) + return result + + +def _build_recommendation_blocked_words( + competitors: list[str], all_names: dict[str, list[str]] +) -> list[dict]: + """Build blocked word entries for competitor recommendations.""" + result = [] + for comp in competitors: + for name in all_names[comp]: + for prefix in ["try", "use", "switch to", "consider"]: + result.append({ + "keyword": f"{prefix} {name}", + "action": "BLOCK", + "description": f"Recommendation to competitor ({comp})", + }) + return result + + +def _build_comparison_blocked_words( + competitors: list[str], all_names: dict[str, list[str]], brand_name: str +) -> list[dict]: + """Build blocked word entries for unfavorable competitor comparisons.""" + result = [] + for comp in competitors: + for name in all_names[comp]: + result.append({ + "keyword": f"{name} is better", + "action": "BLOCK", + "description": f"Unfavorable comparison ({comp})", + }) + + # Brand-level comparisons (only need one entry each, not per-competitor) + result.append({ + "keyword": f"better than {brand_name}", + "action": "BLOCK", + "description": "Unfavorable comparison", + }) + result.append({ + "keyword": f"{brand_name} is worse", + "action": "BLOCK", + "description": "Unfavorable comparison", + }) + + return result diff --git a/policy_templates.json b/policy_templates.json index f27f20187f..0dfa070aaa 100644 --- a/policy_templates.json +++ b/policy_templates.json @@ -1522,7 +1522,13 @@ "guardrails": [ "aviation-ops-data-protection", "aviation-safety-topic-filter", - "airline-brand-protection-filter" + "airline-brand-protection-filter", + "competitor-name-input-blocker", + "competitor-name-output-blocker", + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "complexity": "High", "parameters": [ @@ -1531,9 +1537,14 @@ "label": "Your Airline / Brand Name", "type": "text", "required": true, - "placeholder": "e.g. Emirates" + "placeholder": "e.g. Acme Airlines" } ], + "llm_enrichment": { + "parameter": "brand_name", + "prompt": "List the top 30 direct competitors of {{brand_name}} in the airline industry. Include major international carriers, regional competitors, and low-cost carriers that operate on overlapping routes. Return ONLY airline/brand names, one per line, no numbering, no explanations.", + "result_key": "competitors" + }, "guardrailDefinitions": [ { "guardrail_name": "aviation-ops-data-protection", @@ -1675,6 +1686,72 @@ "guardrail_info": { "description": "Blocks AI-generated fake incident reports, unauthorized statements, and reputation-damaging content about your brand (runs on output)" } + }, + { + "guardrail_name": "competitor-name-input-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs that mention competitor names (pre_call)" + } + }, + { + "guardrail_name": "competitor-name-output-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks AI outputs that mention competitor names (post_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks user requests asking to recommend competitors (pre_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-output-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks AI from recommending or suggesting competitor services (post_call)" + } + }, + { + "guardrail_name": "competitor-comparison-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs requesting unfavorable brand comparisons (pre_call)" + } + }, + { + "guardrail_name": "competitor-comparison-output-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks AI outputs with unfavorable brand comparisons (post_call)" + } } ], "templateData": { @@ -1683,7 +1760,13 @@ "guardrails_add": [ "aviation-ops-data-protection", "aviation-safety-topic-filter", - "airline-brand-protection-filter" + "airline-brand-protection-filter", + "competitor-name-input-blocker", + "competitor-name-output-blocker", + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "guardrails_remove": [] }, @@ -1812,9 +1895,12 @@ "iconColor": "text-orange-500", "iconBg": "bg-orange-50", "guardrails": [ + "competitor-input-blocker", "competitor-output-blocker", - "competitor-recommendation-filter", - "competitor-comparison-filter" + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "complexity": "Medium", "parameters": [ @@ -1823,15 +1909,26 @@ "label": "Your Brand Name", "type": "text", "required": true, - "placeholder": "e.g. Emirates" + "placeholder": "e.g. Acme Airlines" } ], "llm_enrichment": { "parameter": "brand_name", - "prompt": "List the top 10 direct competitors of {{brand_name}} in the same industry. Return ONLY company/brand names, one per line, no numbering, no explanations.", + "prompt": "List the top 30 direct competitors of {{brand_name}} in the same industry. Return ONLY company/brand names, one per line, no numbering, no explanations.", "result_key": "competitors" }, "guardrailDefinitions": [ + { + "guardrail_name": "competitor-input-blocker", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitors_blocked_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs that mention competitor brands (pre_call)" + } + }, { "guardrail_name": "competitor-output-blocker", "litellm_params": { @@ -1840,39 +1937,64 @@ "blocked_words": "{{competitors_blocked_words}}" }, "guardrail_info": { - "description": "Blocks AI outputs that mention or promote competitor brands (auto-discovered via LLM)" + "description": "Blocks AI outputs that mention competitor brands (post_call)" } }, { - "guardrail_name": "competitor-recommendation-filter", + "guardrail_name": "competitor-recommendation-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_recommendation_words}}" + }, + "guardrail_info": { + "description": "Blocks user requests asking to recommend competitors (pre_call)" + } + }, + { + "guardrail_name": "competitor-recommendation-output-filter", "litellm_params": { "guardrail": "litellm_content_filter", "mode": "post_call", "blocked_words": "{{competitor_recommendation_words}}" }, "guardrail_info": { - "description": "Blocks AI from recommending, suggesting, or directing users to competitor services" + "description": "Blocks AI from recommending or suggesting competitor services (post_call)" } }, { - "guardrail_name": "competitor-comparison-filter", + "guardrail_name": "competitor-comparison-input-filter", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "blocked_words": "{{competitor_comparison_words}}" + }, + "guardrail_info": { + "description": "Blocks user inputs requesting unfavorable brand comparisons (pre_call)" + } + }, + { + "guardrail_name": "competitor-comparison-output-filter", "litellm_params": { "guardrail": "litellm_content_filter", "mode": "post_call", "blocked_words": "{{competitor_comparison_words}}" }, "guardrail_info": { - "description": "Blocks unfavorable comparisons between your brand and competitors in AI outputs" + "description": "Blocks AI outputs with unfavorable brand comparisons (post_call)" } } ], "templateData": { "policy_name": "competitor-mention-detection", - "description": "Detects and blocks competitor mentions in AI outputs. Uses LLM-powered competitor discovery based on your brand name.", + "description": "Detects and blocks competitor mentions in both inputs and outputs. Uses LLM-powered competitor discovery based on your brand name.", "guardrails_add": [ + "competitor-input-blocker", "competitor-output-blocker", - "competitor-recommendation-filter", - "competitor-comparison-filter" + "competitor-recommendation-input-filter", + "competitor-recommendation-output-filter", + "competitor-comparison-input-filter", + "competitor-comparison-output-filter" ], "guardrails_remove": [] }, diff --git a/tests/test_litellm/proxy/management_endpoints/test_policy_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_policy_endpoints.py index 2fb2a6dcc8..23754bf326 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_policy_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_policy_endpoints.py @@ -587,3 +587,256 @@ class TestApplyPoliciesDirectGuardrailNames: # Sorted order: direct_guardrail then from_policy; final output is from_policy assert result["inputs"] == first_output assert result["guardrail_errors"] == [] + + +# --------------------------------------------------------------------------- +# Tests for competitor enrichment helper functions +# --------------------------------------------------------------------------- +from litellm.proxy.management_endpoints.policy_endpoints import ( + _build_all_names_per_competitor, + _build_comparison_blocked_words, + _build_competitor_guardrail_definitions, + _build_name_blocked_words, + _build_recommendation_blocked_words, + _build_refinement_prompt, + _clean_competitor_line, + _parse_variations_response, +) + + +class TestCleanCompetitorLine: + """Tests for _clean_competitor_line.""" + + def test_strips_bullets_and_dashes(self): + assert _clean_competitor_line("- United Airlines") == "United Airlines" + assert _clean_competitor_line(" - JetBlue ") == "JetBlue" + + def test_strips_trailing_punctuation(self): + assert _clean_competitor_line("Delta Airlines.") == "Delta Airlines" + assert _clean_competitor_line("Southwest)") == "Southwest" + + def test_returns_none_for_empty(self): + assert _clean_competitor_line("") is None + assert _clean_competitor_line(" ") is None + + def test_returns_none_for_single_char(self): + assert _clean_competitor_line("A") is None + assert _clean_competitor_line(" - ") is None + + def test_plain_name(self): + assert _clean_competitor_line("Qatar Airways") == "Qatar Airways" + + +class TestParseVariationsResponse: + """Tests for _parse_variations_response.""" + + def test_parses_standard_format(self): + raw = "Delta Airlines: Delta Air Lines, DeltaAirlines, Delta\nUnited Airlines: United, UAL" + competitors = ["Delta Airlines", "United Airlines"] + result = _parse_variations_response(raw, competitors) + assert "Delta Airlines" in result + assert "Delta Air Lines" in result["Delta Airlines"] + assert "United" in result["United Airlines"] + + def test_case_insensitive_matching(self): + raw = "delta airlines: Delta Air Lines, DeltaAirlines" + competitors = ["Delta Airlines"] + result = _parse_variations_response(raw, competitors) + assert "Delta Airlines" in result + assert len(result["Delta Airlines"]) == 2 + + def test_skips_lines_without_colon(self): + raw = "This is a header\nDelta Airlines: Delta Air Lines" + competitors = ["Delta Airlines"] + result = _parse_variations_response(raw, competitors) + assert len(result) == 1 + + def test_skips_unknown_competitors(self): + raw = "Unknown Corp: Foo, Bar\nDelta Airlines: Delta" + competitors = ["Delta Airlines"] + result = _parse_variations_response(raw, competitors) + assert "Unknown Corp" not in result + assert "Delta Airlines" in result + + def test_filters_out_self_reference(self): + raw = "Delta Airlines: Delta Airlines, Delta Air Lines" + competitors = ["Delta Airlines"] + result = _parse_variations_response(raw, competitors) + # "Delta Airlines" should be filtered out (same as canonical) + assert "Delta Airlines" not in result["Delta Airlines"] + assert "Delta Air Lines" in result["Delta Airlines"] + + def test_empty_input(self): + assert _parse_variations_response("", []) == {} + + +class TestBuildRefinementPrompt: + """Tests for _build_refinement_prompt.""" + + def test_includes_brand_name(self): + prompt = _build_refinement_prompt("add 10 more", ["Delta"], "Emirates") + assert "Emirates" in prompt + + def test_includes_existing_competitors(self): + prompt = _build_refinement_prompt("add more", ["Delta", "United"], "Emirates") + assert "Delta" in prompt + assert "United" in prompt + + def test_includes_instruction(self): + prompt = _build_refinement_prompt("add 10 from Asia", ["Delta"], "Emirates") + assert "add 10 from Asia" in prompt + + def test_asks_for_new_names_only(self): + prompt = _build_refinement_prompt("add more", ["Delta"], "Emirates") + assert "NEW" in prompt + + +class TestBuildAllNamesPerCompetitor: + """Tests for _build_all_names_per_competitor.""" + + def test_includes_canonical_and_variations(self): + result = _build_all_names_per_competitor( + ["Delta Airlines"], {"Delta Airlines": ["Delta", "DeltaAir"]} + ) + assert result["Delta Airlines"] == ["Delta Airlines", "Delta", "DeltaAir"] + + def test_no_variations(self): + result = _build_all_names_per_competitor(["Delta Airlines"], {}) + assert result["Delta Airlines"] == ["Delta Airlines"] + + def test_multiple_competitors(self): + result = _build_all_names_per_competitor( + ["Delta", "United"], + {"Delta": ["DL"], "United": ["UA"]}, + ) + assert len(result) == 2 + assert result["Delta"] == ["Delta", "DL"] + assert result["United"] == ["United", "UA"] + + +class TestBuildNameBlockedWords: + """Tests for _build_name_blocked_words.""" + + def test_basic_output(self): + all_names = {"Delta": ["Delta", "DL"]} + result = _build_name_blocked_words(["Delta"], all_names) + keywords = [r["keyword"] for r in result] + assert "Delta" in keywords + assert "DL" in keywords + assert all(r["action"] == "BLOCK" for r in result) + + def test_descriptions_differ_for_variations(self): + all_names = {"Delta": ["Delta", "DL"]} + result = _build_name_blocked_words(["Delta"], all_names) + descs = {r["keyword"]: r["description"] for r in result} + assert "Competitor: Delta" == descs["Delta"] + assert "variation" in descs["DL"].lower() + + +class TestBuildRecommendationBlockedWords: + """Tests for _build_recommendation_blocked_words.""" + + def test_generates_prefix_combinations(self): + all_names = {"Delta": ["Delta"]} + result = _build_recommendation_blocked_words(["Delta"], all_names) + keywords = [r["keyword"] for r in result] + assert "try Delta" in keywords + assert "use Delta" in keywords + assert "switch to Delta" in keywords + assert "consider Delta" in keywords + + def test_includes_variations(self): + all_names = {"Delta": ["Delta", "DL"]} + result = _build_recommendation_blocked_words(["Delta"], all_names) + keywords = [r["keyword"] for r in result] + assert "try DL" in keywords + + +class TestBuildComparisonBlockedWords: + """Tests for _build_comparison_blocked_words.""" + + def test_generates_competitor_comparisons(self): + all_names = {"Delta": ["Delta"]} + result = _build_comparison_blocked_words(["Delta"], all_names, "Emirates") + keywords = [r["keyword"] for r in result] + assert "Delta is better" in keywords + + def test_generates_brand_comparisons_once(self): + all_names = {"Delta": ["Delta"], "United": ["United"]} + result = _build_comparison_blocked_words(["Delta", "United"], all_names, "Emirates") + keywords = [r["keyword"] for r in result] + # Brand-level entries should appear exactly once + assert keywords.count("better than Emirates") == 1 + assert keywords.count("Emirates is worse") == 1 + + def test_includes_variation_comparisons(self): + all_names = {"Delta": ["Delta", "DL"]} + result = _build_comparison_blocked_words(["Delta"], all_names, "Emirates") + keywords = [r["keyword"] for r in result] + assert "DL is better" in keywords + + +class TestBuildCompetitorGuardrailDefinitions: + """Tests for _build_competitor_guardrail_definitions.""" + + def test_populates_blocked_words_for_known_guardrail_names(self): + definitions = [ + { + "guardrail_name": "competitor-name-blocker", + "litellm_params": {"blocked_words": []}, + }, + { + "guardrail_name": "competitor-recommendation-filter", + "litellm_params": {"blocked_words": []}, + }, + ] + result = _build_competitor_guardrail_definitions( + definitions, ["Delta"], "Emirates", {"Delta": ["DL"]} + ) + # Name blocker should have entries + name_blocker = next(d for d in result if d["guardrail_name"] == "competitor-name-blocker") + assert len(name_blocker["litellm_params"]["blocked_words"]) > 0 + + # Recommendation filter should have entries + rec_filter = next(d for d in result if d["guardrail_name"] == "competitor-recommendation-filter") + assert len(rec_filter["litellm_params"]["blocked_words"]) > 0 + + def test_does_not_modify_unknown_guardrail_names(self): + definitions = [ + { + "guardrail_name": "some-other-guardrail", + "litellm_params": {"blocked_words": ["original"]}, + }, + ] + result = _build_competitor_guardrail_definitions( + definitions, ["Delta"], "Emirates" + ) + assert result[0]["litellm_params"]["blocked_words"] == ["original"] + + def test_does_not_mutate_original_definitions(self): + definitions = [ + { + "guardrail_name": "competitor-name-blocker", + "litellm_params": {"blocked_words": []}, + }, + ] + _build_competitor_guardrail_definitions(definitions, ["Delta"], "Emirates") + # Original should be unchanged + assert definitions[0]["litellm_params"]["blocked_words"] == [] + + def test_handles_input_and_output_blocker_variants(self): + definitions = [ + { + "guardrail_name": "competitor-name-input-blocker", + "litellm_params": {"blocked_words": []}, + }, + { + "guardrail_name": "competitor-name-output-blocker", + "litellm_params": {"blocked_words": []}, + }, + ] + result = _build_competitor_guardrail_definitions( + definitions, ["Delta"], "Emirates" + ) + for defn in result: + assert len(defn["litellm_params"]["blocked_words"]) > 0 diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index a1d4a70869..b1694369c7 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -5555,19 +5555,24 @@ export const getPolicyTemplates = async (accessToken: string) => { export const enrichPolicyTemplate = async ( accessToken: string, templateId: string, - parameters: Record + parameters: Record, + model?: string, + competitors?: string[] ) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/policy/templates/enrich` : `/policy/templates/enrich`; + const body: any = { template_id: templateId, parameters }; + if (model) body.model = model; + if (competitors) body.competitors = competitors; const response = await fetch(url, { method: "POST", headers: { [globalLitellmHeaderName]: `Bearer ${accessToken}`, "Content-Type": "application/json", }, - body: JSON.stringify({ template_id: templateId, parameters }), + body: JSON.stringify(body), }); if (!response.ok) { @@ -5585,6 +5590,78 @@ export const enrichPolicyTemplate = async ( } }; +export const enrichPolicyTemplateStream = async ( + accessToken: string, + templateId: string, + parameters: Record, + model: string, + onCompetitor: (name: string) => void, + onDone: (result: { + competitors: string[]; + competitor_variations: Record; + guardrailDefinitions: any[]; + }) => void, + onError?: (error: string) => void, + options?: { instruction?: string; existingCompetitors?: string[] }, + onStatus?: (message: string) => void +) => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/policy/templates/enrich/stream` + : `/policy/templates/enrich/stream`; + const body: any = { template_id: templateId, parameters, model }; + if (options?.instruction) body.instruction = options.instruction; + if (options?.existingCompetitors) body.competitors = options.existingCompetitors; + + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const reader = response.body?.getReader(); + if (!reader) throw new Error("No response body"); + + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + try { + const event = JSON.parse(line.slice(6)); + if (event.type === "competitor") { + onCompetitor(event.name); + } else if (event.type === "status") { + onStatus?.(event.message); + } else if (event.type === "done") { + onDone(event); + } else if (event.type === "error") { + onError?.(event.message); + } + } catch { + // skip malformed events + } + } + } +}; + export const createPolicyCall = async (accessToken: string, policyData: any) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/policies` : `/policies`; diff --git a/ui/litellm-dashboard/src/components/policies/guardrail_selection_modal.tsx b/ui/litellm-dashboard/src/components/policies/guardrail_selection_modal.tsx index 2984478ba5..adf1b14a4a 100644 --- a/ui/litellm-dashboard/src/components/policies/guardrail_selection_modal.tsx +++ b/ui/litellm-dashboard/src/components/policies/guardrail_selection_modal.tsx @@ -222,6 +222,31 @@ const GuardrailSelectionModal: React.FC = ({ )} + {/* Discovered Competitors */} + {template?.discoveredCompetitors?.length > 0 && ( + <> + +
+
+ ✨ + + AI-Discovered Competitors ({template.discoveredCompetitors.length}) + +
+
+ {template.discoveredCompetitors.map((name: string) => ( + + {name} + + ))} +
+

+ These competitor names will be automatically blocked by the competitor-name-blocker guardrail. +

+
+ + )} + {/* Selected Summary */} diff --git a/ui/litellm-dashboard/src/components/policies/index.tsx b/ui/litellm-dashboard/src/components/policies/index.tsx index f5fadebdec..01f6d1e13d 100644 --- a/ui/litellm-dashboard/src/components/policies/index.tsx +++ b/ui/litellm-dashboard/src/components/policies/index.tsx @@ -228,7 +228,10 @@ const PoliciesPanel: React.FC = ({ return JSON.parse(templateStr); }; - const handleParameterConfirm = async (parameters: Record) => { + const handleParameterConfirm = async ( + parameters: Record, + enrichmentOptions?: { model?: string; competitors?: string[] } + ) => { if (!accessToken || !pendingTemplate) return; setIsEnrichingTemplate(true); @@ -237,14 +240,20 @@ const PoliciesPanel: React.FC = ({ let enrichedTemplate = pendingTemplate; if (pendingTemplate.llm_enrichment) { - // Call backend to enrich template with LLM-discovered data + // Call backend to enrich template with LLM-discovered data (or user-provided competitors) const enrichResult = await enrichPolicyTemplate( accessToken, pendingTemplate.id, - parameters + parameters, + enrichmentOptions?.model, + enrichmentOptions?.competitors ); - // The backend returns the enriched guardrailDefinitions - enrichedTemplate = { ...pendingTemplate, guardrailDefinitions: enrichResult.guardrailDefinitions }; + // The backend returns the enriched guardrailDefinitions + discovered competitors + enrichedTemplate = { + ...pendingTemplate, + guardrailDefinitions: enrichResult.guardrailDefinitions, + discoveredCompetitors: enrichResult.competitors || [], + }; } // Substitute parameters in template @@ -491,6 +500,7 @@ const PoliciesPanel: React.FC = ({ onConfirm={handleParameterConfirm} onCancel={handleParameterCancel} isLoading={isEnrichingTemplate} + accessToken={accessToken || ""} /> diff --git a/ui/litellm-dashboard/src/components/policies/template_parameter_modal.tsx b/ui/litellm-dashboard/src/components/policies/template_parameter_modal.tsx index ce1e5bc22e..1bbb73db9e 100644 --- a/ui/litellm-dashboard/src/components/policies/template_parameter_modal.tsx +++ b/ui/litellm-dashboard/src/components/policies/template_parameter_modal.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect } from "react"; -import { Modal, Spin } from "antd"; +import { Modal, Spin, Radio, Select } from "antd"; import { Button, TextInput } from "@tremor/react"; +import { modelHubCall, enrichPolicyTemplateStream } from "../networking"; interface TemplateParameter { name: string; @@ -13,9 +14,13 @@ interface TemplateParameter { interface TemplateParameterModalProps { visible: boolean; template: any; - onConfirm: (parameters: Record) => void; + onConfirm: ( + parameters: Record, + enrichmentOptions?: { model?: string; competitors?: string[] } + ) => void; onCancel: () => void; isLoading?: boolean; + accessToken: string; } const TemplateParameterModal: React.FC = ({ @@ -24,10 +29,28 @@ const TemplateParameterModal: React.FC = ({ onConfirm, onCancel, isLoading = false, + accessToken, }) => { const [parameterValues, setParameterValues] = useState>({}); + const [competitorMode, setCompetitorMode] = useState<"ai" | "manual">("ai"); + const [selectedModel, setSelectedModel] = useState(undefined); + const [availableModels, setAvailableModels] = useState([]); + const [isLoadingModels, setIsLoadingModels] = useState(false); + const [competitorTags, setCompetitorTags] = useState([]); + const [variationsMap, setVariationsMap] = useState>({}); + const [isGenerating, setIsGenerating] = useState(false); + const [refinementInput, setRefinementInput] = useState(""); + const [isRefining, setIsRefining] = useState(false); + const [hasGenerated, setHasGenerated] = useState(false); + const [statusMessage, setStatusMessage] = useState(""); const parameters: TemplateParameter[] = template?.parameters || []; + const hasEnrichment = !!template?.llm_enrichment; + const enrichmentParam = hasEnrichment ? template.llm_enrichment.parameter : null; + + const nonEnrichmentParams = hasEnrichment + ? parameters.filter((p) => p.name !== enrichmentParam) + : parameters; useEffect(() => { if (visible && template) { @@ -36,15 +59,136 @@ const TemplateParameterModal: React.FC = ({ initial[p.name] = ""; }); setParameterValues(initial); + setCompetitorMode("ai"); + setSelectedModel(undefined); + setCompetitorTags([]); + setVariationsMap({}); + setIsGenerating(false); + setRefinementInput(""); + setIsRefining(false); + setHasGenerated(false); + setStatusMessage(""); } }, [visible, template]); - const allRequiredFilled = parameters + useEffect(() => { + if (visible && hasEnrichment && competitorMode === "ai" && availableModels.length === 0) { + loadModels(); + } + }, [visible, hasEnrichment, competitorMode]); + + const loadModels = async () => { + if (!accessToken) return; + setIsLoadingModels(true); + try { + const fetchedModels = await modelHubCall(accessToken); + if (fetchedModels?.data?.length > 0) { + const models = fetchedModels.data + .map((item: any) => item.model_group as string) + .sort(); + setAvailableModels(models); + } + } catch (error) { + console.error("Error fetching models:", error); + } finally { + setIsLoadingModels(false); + } + }; + + const handleGenerateNames = async () => { + if (!accessToken || !selectedModel || !template) return; + const brandName = (parameterValues[enrichmentParam || "brand_name"] || "").trim(); + if (!brandName) return; + + setIsGenerating(true); + setCompetitorTags([]); + setVariationsMap({}); + setStatusMessage(""); + try { + await enrichPolicyTemplateStream( + accessToken, + template.id, + parameterValues, + selectedModel, + (name) => { + setCompetitorTags((prev) => [...prev, name]); + }, + (result) => { + setCompetitorTags(result.competitors); + setVariationsMap(result.competitor_variations || {}); + setIsGenerating(false); + setHasGenerated(true); + setStatusMessage(""); + }, + (error) => { + console.error("Streaming error:", error); + setIsGenerating(false); + setStatusMessage(""); + }, + undefined, + (status) => setStatusMessage(status), + ); + } catch (error) { + console.error("Error generating competitor names:", error); + setIsGenerating(false); + } + }; + + const handleRefine = async () => { + if (!accessToken || !selectedModel || !template || !refinementInput.trim()) return; + + setIsRefining(true); + setStatusMessage(""); + try { + await enrichPolicyTemplateStream( + accessToken, + template.id, + parameterValues, + selectedModel, + (name) => { + setCompetitorTags((prev) => { + if (prev.some((t) => t.toLowerCase() === name.toLowerCase())) return prev; + return [...prev, name]; + }); + }, + (result) => { + setCompetitorTags(result.competitors); + setVariationsMap(result.competitor_variations || {}); + setIsRefining(false); + setRefinementInput(""); + setStatusMessage(""); + }, + (error) => { + console.error("Refinement error:", error); + setIsRefining(false); + setStatusMessage(""); + }, + { + instruction: refinementInput.trim(), + existingCompetitors: competitorTags, + }, + (status) => setStatusMessage(status), + ); + } catch (error) { + console.error("Error refining competitor names:", error); + setIsRefining(false); + } + }; + + const allNonEnrichmentFilled = nonEnrichmentParams .filter((p) => p.required) .every((p) => (parameterValues[p.name] || "").trim().length > 0); + const brandNameFilled = enrichmentParam + ? (parameterValues[enrichmentParam] || "").trim().length > 0 + : true; + + const canContinue = hasEnrichment + ? allNonEnrichmentFilled && brandNameFilled && competitorTags.length > 0 + : allNonEnrichmentFilled && brandNameFilled; + const handleConfirm = () => { - onConfirm(parameterValues); + onConfirm(parameterValues, { competitors: competitorTags }); }; return ( @@ -53,15 +197,13 @@ const TemplateParameterModal: React.FC = ({

{template?.title}

- {template?.llm_enrichment - ? "Enter your brand name to auto-discover competitors and configure guardrails" - : "Configure template parameters"} + Configure competitor blocking for your brand

} open={visible} onCancel={onCancel} - width={500} + width={700} footer={[ , ]} >
- {parameters.map((param) => ( + {nonEnrichmentParams.map((param) => (
))} - {template?.llm_enrichment && ( -
-

- This template uses AI to automatically discover your competitors and configure - guardrails. An onboarded LLM will be called to identify competitor names. -

-
+ {hasEnrichment && ( + <> +
+ + setCompetitorMode(e.target.value)} + className="w-full" + > +
+ + ✨ Use AI + + + Enter Manually + +
+ +
+ + {/* Brand Name */} +
+ + + setParameterValues((prev) => ({ + ...prev, + [enrichmentParam || "brand_name"]: e.target.value, + })) + } + /> +
+ + {competitorMode === "ai" && ( + <> +
+ + setCompetitorTags(values)} + tokenSeparators={[","]} + open={false} + suffixIcon={null} + /> +

+ Type a name and press Enter to add. Click ✕ to remove. +

+ {statusMessage && ( +
+ + {statusMessage} +
+ )} + {Object.keys(variationsMap).length > 0 && !statusMessage && ( +

+ ✓ {Object.values(variationsMap).flat().length} alternate spellings & variations auto-generated for guardrail matching +

+ )} +
+ + {/* Refinement input — shown after initial generation in AI mode */} + {competitorMode === "ai" && hasGenerated && competitorTags.length > 0 && ( +
+ +
+ setRefinementInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter" && refinementInput.trim() && !isRefining) { + handleRefine(); + } + }} + disabled={isRefining} + /> + +
+

+ Give instructions to add, remove, or change competitors. Press Enter to send. +

+
+ )} + )} - {isLoading && ( -
- - - {template?.llm_enrichment - ? "Using AI to discover competitors..." - : "Processing template..."} - -
- )} + {!hasEnrichment && + parameters.map((param) => ( +
+ + + setParameterValues((prev) => ({ + ...prev, + [param.name]: e.target.value, + })) + } + /> +
+ ))}
);