diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b067941123..4744ab048c 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: Schedule Demo - url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat + url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM - name: Discord url: https://discord.com/invite/wuPM9dRgDw diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index b2a298bfcd..9477dd2f8e 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -2,47 +2,28 @@ name: Check Duplicate Issues on: issues: - types: [opened] + types: [opened, edited] jobs: - check-duplicates: - if: github.event.action == 'opened' + check-duplicate: runs-on: ubuntu-latest permissions: - contents: read issues: write + contents: read steps: - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - - name: Check duplicates - env: - ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }} - ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} + - name: Check for potential duplicates + uses: wow-actions/potential-duplicates@v1 + with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PROMPT: | - A new issue has been created in the ${{ github.repository }} repository. + label: potential-duplicate + threshold: 0.6 + reaction: eyes + comment: | + **⚠️ Potential duplicate detected** - Issue number: ${{ github.event.issue.number }} + This issue appears similar to existing issue(s): + {{#issues}} + - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar) + {{/issues}} - Lookup this issue with gh issue view ${{ github.event.issue.number }} --repo ${{ github.repository }}. - - Search through existing issues (excluding #${{ github.event.issue.number }}) to find potential duplicates. - - Use gh issue list --repo ${{ github.repository }} with relevant search terms from the new issue's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed issues. - - Consider: - 1. Similar titles or descriptions - 2. Same error messages or symptoms - 3. Related functionality or components - 4. Similar feature requests - - If you find potential duplicates, post a SINGLE comment on issue #${{ github.event.issue.number }} using gh issue comment ${{ github.event.issue.number }} --repo ${{ github.repository }} with this format: - - _This comment was generated by an LLM and may be inaccurate._ - - This issue might be a duplicate of existing issues. Please check: - - #[issue_number]: [brief description of similarity] - - If you find NO duplicates, do NOT post any comment. Stay silent. - run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh issue *)" + Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. diff --git a/.github/workflows/check_duplicate_prs.yml b/.github/workflows/check_duplicate_prs.yml deleted file mode 100644 index 5a5f1a89e6..0000000000 --- a/.github/workflows/check_duplicate_prs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Check Duplicate PRs - -on: - pull_request_target: - types: [opened] - -jobs: - check-duplicates: - if: | - github.event.pull_request.user.login != 'ishaan-jaff' && - github.event.pull_request.user.login != 'krrishdholakia' && - github.event.pull_request.user.login != 'actions-user' && - !endsWith(github.event.pull_request.user.login, '[bot]') - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - - name: Check duplicates - env: - ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }} - ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PROMPT: | - A new PR has been opened in the ${{ github.repository }} repository. - - PR number: ${{ github.event.pull_request.number }} - - Lookup this PR with gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }}. - - Search through existing open PRs (excluding #${{ github.event.pull_request.number }}) to find potential duplicates. - - Use gh pr list --repo ${{ github.repository }} with relevant search terms from the new PR's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed PRs. - - Consider: - 1. Similar titles or descriptions - 2. Same bug fix or feature being implemented - 3. Related functionality or components - 4. Overlapping code changes (same files or areas) - - If you find potential duplicates, post a SINGLE comment on PR #${{ github.event.pull_request.number }} using gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} with this format: - - _This comment was generated by an LLM and may be inaccurate._ - - This PR might be a duplicate of existing PRs. Please check: - - #[pr_number]: [brief description of similarity] - - If you find NO duplicates, do NOT post any comment. Stay silent. - run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh pr *)" diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index 0b5df73862..348ff300ff 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -123,7 +123,7 @@ if __name__ == "__main__": + docker_run_command + "\n\n" + "### Don't want to maintain your internal proxy? get in touch πŸŽ‰" - + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" + + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" + "\n\n" + "## Load Test LiteLLM Proxy Results" + "\n\n" diff --git a/README.md b/README.md index 7790c67afd..3ebaefb10c 100644 --- a/README.md +++ b/README.md @@ -399,7 +399,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Enterprise For companies that need better security, user management and professional support -[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) This covers: - βœ… **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):** diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md index a543d91011..57115eb96a 100644 --- a/cookbook/benchmark/readme.md +++ b/cookbook/benchmark/readme.md @@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?': ``` ## Support -**🀝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🀝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/gollem_go_agent_framework/README.md b/cookbook/gollem_go_agent_framework/README.md new file mode 100644 index 0000000000..729f985d08 --- /dev/null +++ b/cookbook/gollem_go_agent_framework/README.md @@ -0,0 +1,119 @@ +# Gollem Go Agent Framework with LiteLLM + +A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output. + +## Quick Start + +### 1. Start LiteLLM Proxy + +```bash +# Simple start with a single model +litellm --model gpt-4o + +# Or with the example config for multi-provider access +litellm --config proxy_config.yaml +``` + +### 2. Run the examples + +```bash +# Install Go dependencies +go mod tidy + +# Basic agent +go run ./basic + +# Agent with type-safe tools +go run ./tools + +# Streaming responses +go run ./streaming +``` + +## Configuration + +The included `proxy_config.yaml` sets up three providers through LiteLLM: + +```yaml +model_list: + - model_name: gpt-4o # OpenAI + - model_name: claude-sonnet # Anthropic + - model_name: gemini-pro # Google Vertex AI +``` + +Switch providers in Go by changing a single string β€” no code changes needed: + +```go +model := openai.NewLiteLLM("http://localhost:4000", + openai.WithModel("gpt-4o"), // OpenAI + // openai.WithModel("claude-sonnet"), // Anthropic + // openai.WithModel("gemini-pro"), // Google +) +``` + +## Examples + +### `basic/` β€” Basic Agent + +Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation. + +### `tools/` β€” Type-Safe Tools + +Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags β€” the schema is generated automatically at compile time. + +### `streaming/` β€” Streaming Responses + +Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough. + +## How It Works + +Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box: + +- **Chat completions** β€” standard request/response +- **Tool use** β€” LiteLLM passes tool definitions and calls through transparently +- **Streaming** β€” Server-Sent Events proxied through LiteLLM +- **Structured output** β€” JSON schema response format works with supporting models + +``` +Go App (gollem) β†’ LiteLLM Proxy β†’ OpenAI / Anthropic / Google / ... +``` + +## Why Use This? + +- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration β€” no runtime surprises +- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string +- **Zero-dependency core**: gollem's core has no external dependencies β€” just stdlib +- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed +- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer + +## Environment Variables + +```bash +# Required for providers you want to use (set in LiteLLM config or env) +export OPENAI_API_KEY="sk-..." +export ANTHROPIC_API_KEY="sk-ant-..." + +# Optional: point to a non-default LiteLLM proxy +export LITELLM_PROXY_URL="http://localhost:4000" +``` + +## Troubleshooting + +**Connection errors?** +- Make sure LiteLLM is running: `litellm --model gpt-4o` +- Check the URL is correct (default: `http://localhost:4000`) + +**Model not found?** +- Verify the model name matches what's configured in LiteLLM +- Run `curl http://localhost:4000/models` to see available models + +**Tool calls not working?** +- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini) +- Check LiteLLM logs for any provider-specific errors + +## Learn More + +- [gollem GitHub](https://github.com/fugue-labs/gollem) +- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core) +- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy) +- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers) diff --git a/cookbook/gollem_go_agent_framework/basic/main.go b/cookbook/gollem_go_agent_framework/basic/main.go new file mode 100644 index 0000000000..838149a8ff --- /dev/null +++ b/cookbook/gollem_go_agent_framework/basic/main.go @@ -0,0 +1,41 @@ +// Basic gollem agent connected to a LiteLLM proxy. +// +// Usage: +// +// litellm --model gpt-4o # start proxy in another terminal +// go run ./basic +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/fugue-labs/gollem/core" + "github.com/fugue-labs/gollem/provider/openai" +) + +func main() { + proxyURL := "http://localhost:4000" + if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { + proxyURL = u + } + + // Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible + // provider pointed at the given URL. + model := openai.NewLiteLLM(proxyURL, + openai.WithModel("gpt-4o"), // any model name configured in LiteLLM + ) + + // Create and run a simple agent. + agent := core.NewAgent[string](model, + core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."), + ) + + result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.") + if err != nil { + log.Fatal(err) + } + fmt.Println(result.Output) +} diff --git a/cookbook/gollem_go_agent_framework/go.mod b/cookbook/gollem_go_agent_framework/go.mod new file mode 100644 index 0000000000..89d9033aa2 --- /dev/null +++ b/cookbook/gollem_go_agent_framework/go.mod @@ -0,0 +1,5 @@ +module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework + +go 1.25.1 + +require github.com/fugue-labs/gollem v0.1.0 diff --git a/cookbook/gollem_go_agent_framework/go.sum b/cookbook/gollem_go_agent_framework/go.sum new file mode 100644 index 0000000000..1eb6c5ac9f --- /dev/null +++ b/cookbook/gollem_go_agent_framework/go.sum @@ -0,0 +1,2 @@ +github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8= +github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ= diff --git a/cookbook/gollem_go_agent_framework/proxy_config.yaml b/cookbook/gollem_go_agent_framework/proxy_config.yaml new file mode 100644 index 0000000000..18265a002b --- /dev/null +++ b/cookbook/gollem_go_agent_framework/proxy_config.yaml @@ -0,0 +1,16 @@ +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-20250514 + api_key: os.environ/ANTHROPIC_API_KEY + + - model_name: gemini-pro + litellm_params: + model: vertex_ai/gemini-2.0-flash + vertex_project: my-project + vertex_location: us-central1 diff --git a/cookbook/gollem_go_agent_framework/streaming/main.go b/cookbook/gollem_go_agent_framework/streaming/main.go new file mode 100644 index 0000000000..42bc9bbe34 --- /dev/null +++ b/cookbook/gollem_go_agent_framework/streaming/main.go @@ -0,0 +1,56 @@ +// Streaming responses from gollem through LiteLLM. +// +// Uses Go 1.23+ range-over-function iterators for real-time token +// streaming via LiteLLM's SSE passthrough. +// +// Usage: +// +// litellm --model gpt-4o +// go run ./streaming +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/fugue-labs/gollem/core" + "github.com/fugue-labs/gollem/provider/openai" +) + +func main() { + proxyURL := "http://localhost:4000" + if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { + proxyURL = u + } + + model := openai.NewLiteLLM(proxyURL, + openai.WithModel("gpt-4o"), + ) + + agent := core.NewAgent[string](model) + + // RunStream returns a streaming result that yields tokens as they arrive. + stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems") + if err != nil { + log.Fatal(err) + } + + // StreamText yields text chunks in real-time. + // The boolean argument controls whether deltas (true) or accumulated + // text (false) is returned. + fmt.Print("Response: ") + for text, err := range stream.StreamText(true) { + if err != nil { + log.Fatal(err) + } + fmt.Print(text) + } + fmt.Println() + + // After streaming completes, the final response is available. + resp := stream.Response() + fmt.Printf("\nTokens used: input=%d, output=%d\n", + resp.Usage.InputTokens, resp.Usage.OutputTokens) +} diff --git a/cookbook/gollem_go_agent_framework/tools/main.go b/cookbook/gollem_go_agent_framework/tools/main.go new file mode 100644 index 0000000000..ed41a95ffe --- /dev/null +++ b/cookbook/gollem_go_agent_framework/tools/main.go @@ -0,0 +1,64 @@ +// Gollem agent with type-safe tools through LiteLLM. +// +// The tool parameters are Go structs β€” gollem generates the JSON schema +// automatically at compile time. LiteLLM passes tool definitions through +// transparently to the underlying provider. +// +// Usage: +// +// litellm --model gpt-4o +// go run ./tools +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/fugue-labs/gollem/core" + "github.com/fugue-labs/gollem/provider/openai" +) + +// WeatherParams defines the tool's input schema via struct tags. +// The JSON schema is generated at compile time β€” no runtime reflection needed. +type WeatherParams struct { + City string `json:"city" description:"City name to get weather for"` + Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"` +} + +func main() { + proxyURL := "http://localhost:4000" + if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { + proxyURL = u + } + + model := openai.NewLiteLLM(proxyURL, + openai.WithModel("gpt-4o"), + ) + + // Define a type-safe tool. The function signature enforces correct types. + weatherTool := core.FuncTool[WeatherParams]( + "get_weather", + "Get current weather for a city", + func(ctx context.Context, p WeatherParams) (string, error) { + unit := p.Unit + if unit == "" { + unit = "fahrenheit" + } + // In production, call a real weather API here. + return fmt.Sprintf("Weather in %s: 72Β°F (22Β°C), sunny", p.City), nil + }, + ) + + agent := core.NewAgent[string](model, + core.WithTools[string](weatherTool), + core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."), + ) + + result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?") + if err != nil { + log.Fatal(err) + } + fmt.Println(result.Output) +} diff --git a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md new file mode 100644 index 0000000000..f6172cd674 --- /dev/null +++ b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md @@ -0,0 +1,147 @@ +--- +slug: anthropic-wildcard-model-access-incident +title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload" +date: 2026-02-23T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [incident-report, proxy, auth, model-access] +hide_table_of_contents: false +--- + +**Date:** Feb 23, 2026 +**Duration:** ~3 hours +**Severity:** High (for users with provider wildcard access rules) +**Status:** Resolved + +## Summary + +When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with: + +``` +key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6. +``` + +The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it. + +- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401. +- **Existing models:** Unaffected β€” only models missing from the stale provider set were impacted. +- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`). + +{/* truncate */} + +--- + +## Background + +LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`: + +```mermaid +flowchart TD + A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model? + proxy/auth/auth_checks.py"] + B --> C["3. Key has models=['anthropic/*'] + β†’ wildcard match attempted"] + C --> D["4. get_llm_provider('claude-sonnet-4-6') + checks litellm.anthropic_models set"] + D -->|"model IN set"| E["5a. βœ… Provider = 'anthropic' + β†’ 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"] + D -->|"model NOT IN set"| F["5b. ❌ Provider unknown + β†’ exception raised β†’ wildcard returns False"] + E --> G["6. Request allowed"] + F --> H["6. 401: key not allowed to access model"] + + style E fill:#d4edda,stroke:#28a745 + style F fill:#f8d7da,stroke:#dc3545 + style H fill:#f8d7da,stroke:#dc3545 + style D fill:#fff3cd,stroke:#ffc107 +``` + +`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`. + +--- + +## Root Cause + +`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again: + +```python +# Before the fix β€” both reload paths looked like this: +new_model_cost_map = get_model_cost_map(url=model_cost_map_url) +litellm.model_cost = new_model_cost_map # βœ… cost map updated +_invalidate_model_cost_lowercase_map() # βœ… cache cleared +# ❌ add_known_models() never called +# β†’ litellm.anthropic_models still has the old set +# β†’ new model not in the set +# β†’ get_llm_provider() raises for the new model +# β†’ wildcard match returns False +# β†’ 401 for every request to the new model +``` + +The gap existed in two places: +1. `_check_and_reload_model_cost_map` β€” the periodic automatic reload (every 10 s) +2. The `/reload/model_cost_map` admin endpoint β€” the manual reload + +**Timeline:** + +1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json` +2. Admin triggers cost map reload via UI β†’ `litellm.model_cost` updated +3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6` +4. `get_llm_provider('claude-sonnet-4-6')` raises β†’ wildcard returns False β†’ 401 +5. Admin reloads cost map again β€” same result (root cause not addressed) +6. ~3 hours of investigation β†’ root cause identified β†’ fix deployed + +--- + +## The Fix + +After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated: + +```python +# After the fix β€” both reload paths now do: +new_model_cost_map = get_model_cost_map(url=model_cost_map_url) +litellm.model_cost = new_model_cost_map +_invalidate_model_cost_lowercase_map() +litellm.add_known_models(model_cost_map=new_model_cost_map) # βœ… sets repopulated +``` + +`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference: + +```python +# Before +def add_known_models(): + for key, value in model_cost.items(): # reads module global β€” ambiguous after reload + ... + +# After +def add_known_models(model_cost_map: Optional[Dict] = None): + _map = model_cost_map if model_cost_map is not None else model_cost + for key, value in _map.items(): # always iterates the map you just fetched + ... +``` + +After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart. + +--- + +## Remediation + +| # | Action | Status | Code | +|---|---|---|---| +| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | βœ… Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) | +| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | βœ… Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) | +| 3 | Update `add_known_models()` to accept an explicit map parameter | βœ… Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) | +| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | βœ… Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) | +| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | βœ… Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) | + +--- diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 1f818cef49..5ed2263d05 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -5,6 +5,44 @@ import Image from '@theme/IdealImage'; Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint. +## Setting Up Benchmarking with Network Mock + +The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider. + +**1. Create a proxy config:** + +```yaml +model_list: + - model_name: db-openai-endpoint + litellm_params: + model: openai/gpt-4o + api_key: "sk-fake-key" + api_base: "https://api.openai.com" + +litellm_settings: + network_mock: true + callbacks: [] + num_retries: 0 + request_timeout: 30 + +general_settings: + master_key: "sk-1234" +``` + +**2. Start the proxy:** + +```bash +litellm --config benchmark_config.yaml --port 4000 --num_workers 8 +``` + +**3. Run the benchmark script:** + +```bash +python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3 +``` + +This measures pure proxy overhead on the hot path without any network latency to a real or fake provider. + ## Setting Up a Fake OpenAI Endpoint For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides: diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index 37fb8bc360..6f81da9105 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -297,6 +297,7 @@ litellm.cache = Cache( similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here + qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used ) response1 = completion( @@ -635,6 +636,7 @@ def __init__( qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model="text-embedding-ada-002", + qdrant_semantic_cache_vector_size: Optional[int] = None, **kwargs ): ``` diff --git a/docs/my-website/docs/contributing.md b/docs/my-website/docs/contributing.md index be7222f6cb..168d092ddc 100644 --- a/docs/my-website/docs/contributing.md +++ b/docs/my-website/docs/contributing.md @@ -79,7 +79,27 @@ cp -r out/* ../../litellm/proxy/_experimental/out/ Then restart the proxy and access the UI at `http://localhost:4000/ui` -## 4. Submitting a PR +## 4. Pre-PR Checklist + +Before submitting your pull request, make sure the following pass locally from `ui/litellm-dashboard/`: + +**Run tests related to your changes:** + +```bash +npx vitest run src/components/path/to/YourComponent.test.tsx +``` + +Tests are co-located with components (e.g., `TeamInfo.tsx` β†’ `TeamInfo.test.tsx`). If you add a new component, add a corresponding `.test.tsx` file next to it. + +**Run the build:** + +```bash +npm run build +``` + +These map to the `ui_tests` and `ui_build` CI checks. + +## 5. Submitting a PR 1. Create a new branch for your changes: ```bash diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 0a1b47f062..6dccf7ff4e 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -4,7 +4,7 @@ import Image from '@theme/IdealImage'; :::info - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) -- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs. +- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs. ::: For companies that need SSO, user management and professional support for LiteLLM Proxy @@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o ### What’s the cost of the Self-Managed Enterprise edition? -Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ### How does deployment with Enterprise License work? @@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr Pricing is based on usage. We can figure out a price that works for your team, on the call. -[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index 2779a478f8..d0bd98a76f 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem'; :::info -This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/interactions.md b/docs/my-website/docs/interactions.md index 32c82a1589..8014bf0536 100644 --- a/docs/my-website/docs/interactions.md +++ b/docs/my-website/docs/interactions.md @@ -130,13 +130,12 @@ Point the Google GenAI SDK to LiteLLM Proxy: ```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy" from google import genai -import os # Point SDK to LiteLLM Proxy -os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000" -os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key - -client = genai.Client() +client = genai.Client( + api_key="sk-1234", # Your LiteLLM API key + http_options={"base_url": "http://localhost:4000"}, +) # Create an interaction interaction = client.interactions.create( @@ -151,12 +150,11 @@ print(interaction.outputs[-1].text) ```python showLineNumbers title="Google GenAI SDK Streaming" from google import genai -import os -os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000" -os.environ["GEMINI_API_KEY"] = "sk-1234" - -client = genai.Client() +client = genai.Client( + api_key="sk-1234", # Your LiteLLM API key + http_options={"base_url": "http://localhost:4000"}, +) for chunk in client.interactions.create_stream( model="gemini/gemini-2.5-flash", diff --git a/docs/my-website/docs/observability/gcs_bucket_integration.md b/docs/my-website/docs/observability/gcs_bucket_integration.md index 4050970808..69b956950e 100644 --- a/docs/my-website/docs/observability/gcs_bucket_integration.md +++ b/docs/my-website/docs/observability/gcs_bucket_integration.md @@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/pass_through/google_ai_studio.md b/docs/my-website/docs/pass_through/google_ai_studio.md index 3de7c54aa7..d87c17fa7e 100644 --- a/docs/my-website/docs/pass_through/google_ai_studio.md +++ b/docs/my-website/docs/pass_through/google_ai_studio.md @@ -35,26 +35,25 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key= ``` - + ```javascript -const { GoogleGenerativeAI } = require("@google/generative-ai"); +const { GoogleGenAI } = require("@google/genai"); -const modelParams = { - model: 'gemini-pro', -}; - -const requestOptions = { - baseUrl: 'http://localhost:4000/gemini', // http:///gemini -}; - -const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key -const model = genAI.getGenerativeModel(modelParams, requestOptions); +const ai = new GoogleGenAI({ + apiKey: "sk-1234", // litellm proxy API key + httpOptions: { + baseUrl: "http://localhost:4000/gemini", // http:///gemini + }, +}); async function main() { try { - const result = await model.generateContent("Explain how AI works"); - console.log(result.response.text()); + const response = await ai.models.generateContent({ + model: "gemini-2.5-flash", + contents: "Explain how AI works", + }); + console.log(response.text); } catch (error) { console.error('Error:', error); } @@ -63,12 +62,13 @@ async function main() { // For streaming responses async function main_streaming() { try { - const streamingResult = await model.generateContentStream("Explain how AI works"); - for await (const chunk of streamingResult.stream) { - console.log('Stream chunk:', JSON.stringify(chunk)); + const response = await ai.models.generateContentStream({ + model: "gemini-2.5-flash", + contents: "Explain how AI works", + }); + for await (const chunk of response) { + process.stdout.write(chunk.text); } - const aggregatedResponse = await streamingResult.response; - console.log('Aggregated response:', JSON.stringify(aggregatedResponse)); } catch (error) { console.error('Error:', error); } @@ -321,29 +321,28 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent? ``` - + ```javascript -const { GoogleGenerativeAI } = require("@google/generative-ai"); +const { GoogleGenAI } = require("@google/genai"); -const modelParams = { - model: 'gemini-pro', -}; - -const requestOptions = { - baseUrl: 'http://localhost:4000/gemini', // http:///gemini - customHeaders: { - "tags": "gemini-js-sdk,pass-through-endpoint" - } -}; - -const genAI = new GoogleGenerativeAI("sk-1234"); -const model = genAI.getGenerativeModel(modelParams, requestOptions); +const ai = new GoogleGenAI({ + apiKey: "sk-1234", + httpOptions: { + baseUrl: "http://localhost:4000/gemini", // http:///gemini + headers: { + "tags": "gemini-js-sdk,pass-through-endpoint", + }, + }, +}); async function main() { try { - const result = await model.generateContent("Explain how AI works"); - console.log(result.response.text()); + const response = await ai.models.generateContent({ + model: "gemini-2.5-flash", + contents: "Explain how AI works", + }); + console.log(response.text); } catch (error) { console.error('Error:', error); } diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 38d6d47be4..e9afe2d993 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -438,6 +438,59 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \ - `event_message` *str*: A human-readable description of the event. +### Digest Mode (Reducing Alert Noise) + +By default, LiteLLM sends a separate Slack message for **every** alert event. For high-frequency alert types like `llm_requests_hanging` or `llm_too_slow`, this can produce hundreds of duplicate messages per day. + +**Digest mode** aggregates duplicate alerts within a configurable time window and emits a single summary message with the total count and time range. + +#### Configuration + +Use `alert_type_config` in `general_settings` to enable digest mode per alert type: + +```yaml +general_settings: + alerting: ["slack"] + alert_type_config: + llm_requests_hanging: + digest: true + digest_interval: 86400 # 24 hours (default) + llm_too_slow: + digest: true + digest_interval: 3600 # 1 hour + llm_exceptions: + digest: true + # uses default interval (86400 seconds / 24 hours) +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `digest` | bool | `false` | Enable digest mode for this alert type | +| `digest_interval` | int | `86400` (24h) | Time window in seconds. Alerts are aggregated within this interval. | + +#### How It Works + +1. When an alert fires for a digest-enabled type, it is **grouped** by `(alert_type, request_model, api_base)` instead of being sent immediately +2. A counter tracks how many times the alert fires within the interval +3. When the interval expires, a **single summary message** is sent: + +``` +Alert type: `llm_requests_hanging` (Digest) +Level: `Medium` +Start: `2026-02-19 03:27:39` +End: `2026-02-20 03:27:39` +Count: `847` + +Message: `Requests are hanging - 600s+ request time` +Request Model: `gemini-2.5-flash` +API Base: `None` +``` + +#### Limitations + +- **Per-instance**: Digest state is held in memory per proxy instance. If you run multiple instances (e.g., Cloud Run with autoscaling), each instance maintains its own digest and emits its own summary. +- **Not durable**: If an instance is terminated before the digest interval expires, the aggregated alerts for that instance are lost. + ## Region-outage alerting (✨ Enterprise feature) :::info diff --git a/docs/my-website/docs/proxy/budget_reset_and_tz.md b/docs/my-website/docs/proxy/budget_reset_and_tz.md index 340e33afe1..0fedff8be1 100644 --- a/docs/my-website/docs/proxy/budget_reset_and_tz.md +++ b/docs/my-website/docs/proxy/budget_reset_and_tz.md @@ -22,6 +22,8 @@ litellm_settings: This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC. If no timezone is specified, UTC will be used by default. +Any valid [IANA timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) is supported (powered by Python's `zoneinfo` module). DST transitions are handled automatically. + Common timezone values: - `UTC` - Coordinated Universal Time diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index 3cb9e9f3fe..3357dcb28b 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -340,6 +340,7 @@ litellm_settings: qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list qdrant_collection_name: test_collection qdrant_quantization_config: binary + qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality similarity_threshold: 0.8 # similarity threshold for semantic cache ``` diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 809478bb6d..2a85fc71b8 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -73,6 +73,7 @@ litellm_settings: qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list qdrant_collection_name: test_collection qdrant_quantization_config: binary + qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality similarity_threshold: 0.8 # similarity threshold for semantic cache # Optional - S3 Cache Settings diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 26a4920c09..baebdf0f31 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -161,7 +161,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints :::info -Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index ad158cb342..86a79cbcfc 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with: :::info -Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 26d2587320..4b525837a2 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; # ✨ Enterprise Features :::tip -To get a license, get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/guardrails/aporia_api.md b/docs/my-website/docs/proxy/guardrails/aporia_api.md index 8c5c1ec194..ceafc19a1c 100644 --- a/docs/my-website/docs/proxy/guardrails/aporia_api.md +++ b/docs/my-website/docs/proxy/guardrails/aporia_api.md @@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index 365fdf81aa..c9115cf826 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -409,7 +409,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md index ddeccaf16d..55d586aee7 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrails_ai.md +++ b/docs/my-website/docs/proxy/guardrails/guardrails_ai.md @@ -59,7 +59,7 @@ curl -i http://localhost:4000/v1/chat/completions \ :::info -✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/guardrails/noma_security.md b/docs/my-website/docs/proxy/guardrails/noma_security.md index a66788cbb5..a397efeb14 100644 --- a/docs/my-website/docs/proxy/guardrails/noma_security.md +++ b/docs/my-website/docs/proxy/guardrails/noma_security.md @@ -6,6 +6,108 @@ import TabItem from '@theme/TabItem'; Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails. +:::warning Deprecated: `guardrail: noma` (Legacy) +`guardrail: noma` is deprecated and users should migrate to `guardrail: noma_v2`. +The legacy `guardrail: noma` API will no longer be supported after March 31, 2026. + +For easier migration of existing integrations, keep `guardrail: noma` and set `use_v2: true`. +With `use_v2: true`, requests route to `noma_v2`; `monitor_mode` and `block_failures` still apply, while `anonymize_input` is ignored. +::: + +## Noma v2 guardrails (Recommended) + +### Quick Start + +```yaml showLineNumbers title="litellm config.yaml" +guardrails: + - guardrail_name: "noma-v2-guard" + litellm_params: + guardrail: noma_v2 + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + api_base: os.environ/NOMA_API_BASE +``` + +If you want to migrate gradually without changing guardrail names yet: + +```yaml showLineNumbers title="litellm config.yaml" +guardrails: + - guardrail_name: "noma-guard" + litellm_params: + guardrail: noma + use_v2: true + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + api_base: os.environ/NOMA_API_BASE +``` + +### Supported Params + +- **`guardrail`**: Use `noma_v2` (recommended), or `noma` with `use_v2: true` for migration +- **`mode`**: `pre_call`, `post_call`, `during_call`, `pre_mcp_call`, `during_mcp_call` +- **`api_key`**: Noma API key (required for Noma SaaS, optional for self-managed deployments) +- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`) +- **`application_id`**: Application identifier. If omitted, v2 checks dynamic `extra_body.application_id`, then configured/env `application_id`; otherwise it is omitted. +- **`monitor_mode`**: If `true`, runs in monitor-only mode without blocking (defaults to `false`) +- **`block_failures`**: If `true`, fail-closed on guardrail technical failures (defaults to `true`) +- **`use_v2`**: Migration toggle when `guardrail: noma` is used + +### Environment Variables + +```shell +export NOMA_API_KEY="your-api-key-here" +export NOMA_API_BASE="https://api.noma.security/" # Optional +export NOMA_APPLICATION_ID="my-app" # Optional +export NOMA_MONITOR_MODE="false" # Optional +export NOMA_BLOCK_FAILURES="true" # Optional +``` + +### Multiple Guardrails + +Apply different v2 configurations for input and output: + +```yaml showLineNumbers title="litellm config.yaml" +guardrails: + - guardrail_name: "noma-v2-input" + litellm_params: + guardrail: noma_v2 + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + + - guardrail_name: "noma-v2-output" + litellm_params: + guardrail: noma_v2 + mode: "post_call" + api_key: os.environ/NOMA_API_KEY +``` + +### Pass Additional Parameters + +This is supported in v2 via `extra_body`. +Currently, `noma_v2` consumes dynamic `application_id`. + +```shell showLineNumbers title="Curl Request" +curl 'http://0.0.0.0:4000/v1/chat/completions' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "guardrails": { + "noma-v2-guard": { + "extra_body": { + "application_id": "my-specific-app-id" + } + } + } + }' +``` +## Noma guardrails (Legacy) + ## Quick Start ### 1. Define Guardrails on your LiteLLM config.yaml diff --git a/docs/my-website/docs/proxy/ip_address.md b/docs/my-website/docs/proxy/ip_address.md index 80d5561da4..8f042d9f18 100644 --- a/docs/my-website/docs/proxy/ip_address.md +++ b/docs/my-website/docs/proxy/ip_address.md @@ -3,7 +3,7 @@ :::info -You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat), to get one today! +You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions), to get one today! ::: diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 1abb127dfd..74a79776fb 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1109,7 +1109,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: @@ -1194,7 +1194,7 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: @@ -1497,7 +1497,7 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md index cf122f85b9..8d39674df1 100644 --- a/docs/my-website/docs/proxy/multiple_admins.md +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -20,7 +20,7 @@ LiteLLM tracks changes to the following entities and actions: :::tip -Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index ec076d8fae..41c4110e44 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -4,7 +4,7 @@ Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` req :::info -This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)) +This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)) ::: diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 994788a3ad..26cb484cbe 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -47,7 +47,7 @@ export LITELLM_LOG="ERROR" :::info -Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/public_routes.md b/docs/my-website/docs/proxy/public_routes.md index 21a92a00be..d5f3941751 100644 --- a/docs/my-website/docs/proxy/public_routes.md +++ b/docs/my-website/docs/proxy/public_routes.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; :::info -Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat). +Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions). ::: diff --git a/docs/my-website/docs/proxy/tag_routing.md b/docs/my-website/docs/proxy/tag_routing.md index 838b2a09d7..399c43d2c0 100644 --- a/docs/my-website/docs/proxy/tag_routing.md +++ b/docs/my-website/docs/proxy/tag_routing.md @@ -215,7 +215,7 @@ LiteLLM Proxy supports team-based tag routing, allowing you to associate specifi :::info -This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index bb35839bb2..2ad7e2a4a8 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -26,7 +26,7 @@ Team 3 -> Disabled Logging (for GDPR compliance) :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: @@ -248,7 +248,7 @@ Use the `/key/generate` or `/key/update` endpoints to add logging callbacks to a :::info -✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index a8a6878fd5..7db59a3300 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -5,7 +5,7 @@ This is an Enterprise feature. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 78cd144d56..e8634f0faf 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -11,7 +11,7 @@ Use JWT's to auth admins / users / projects into the proxy. [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 21eb639581..c5c8031147 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/aws_kms.md b/docs/my-website/docs/secret_managers/aws_kms.md index 79dc80897f..7f69d91fe8 100644 --- a/docs/my-website/docs/secret_managers/aws_kms.md +++ b/docs/my-website/docs/secret_managers/aws_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index 5b7ab1e3e7..c49797a15d 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/azure_key_vault.md b/docs/my-website/docs/secret_managers/azure_key_vault.md index 6ec95b378b..81aeaa3215 100644 --- a/docs/my-website/docs/secret_managers/azure_key_vault.md +++ b/docs/my-website/docs/secret_managers/azure_key_vault.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/cyberark.md b/docs/my-website/docs/secret_managers/cyberark.md index c33aa28670..0a17c0afc3 100644 --- a/docs/my-website/docs/secret_managers/cyberark.md +++ b/docs/my-website/docs/secret_managers/cyberark.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/google_kms.md b/docs/my-website/docs/secret_managers/google_kms.md index 0c6f66846f..31fd6195bd 100644 --- a/docs/my-website/docs/secret_managers/google_kms.md +++ b/docs/my-website/docs/secret_managers/google_kms.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/google_secret_manager.md b/docs/my-website/docs/secret_managers/google_secret_manager.md index a545e7a85b..81878b7e39 100644 --- a/docs/my-website/docs/secret_managers/google_secret_manager.md +++ b/docs/my-website/docs/secret_managers/google_secret_manager.md @@ -6,7 +6,7 @@ [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/hashicorp_vault.md b/docs/my-website/docs/secret_managers/hashicorp_vault.md index e9e0116f4f..52d9b55620 100644 --- a/docs/my-website/docs/secret_managers/hashicorp_vault.md +++ b/docs/my-website/docs/secret_managers/hashicorp_vault.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/secret_managers/overview.md b/docs/my-website/docs/secret_managers/overview.md index a987c72d76..bf7386ab89 100644 --- a/docs/my-website/docs/secret_managers/overview.md +++ b/docs/my-website/docs/secret_managers/overview.md @@ -8,7 +8,7 @@ import Image from '@theme/IdealImage'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) ::: diff --git a/docs/my-website/docs/tutorials/compare_llms.md b/docs/my-website/docs/tutorials/compare_llms.md index d7fdf8d7d9..02877b4660 100644 --- a/docs/my-website/docs/tutorials/compare_llms.md +++ b/docs/my-website/docs/tutorials/compare_llms.md @@ -82,7 +82,7 @@ Benchmark Results for 'When will BerriAI IPO?': +-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ ``` ## Support -**🀝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. +**🀝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. B[Stream Abandoned] + B --> C{Connection cleaned up?} + C -->|Before| D["❌ No β€” connection leaked"] + C -->|After| E["βœ… Yes β€” connection returned to pool"] +``` + +**Redis Connection Pool Reliability** + +Fixed 4 separate connection pool bugs to make how we use Redis more reliable. The most important change was on pools being leaked on cache expiry and the other fixes are detailed here in [PR #21717](https://github.com/BerriAI/litellm/pull/21717). + +```mermaid +graph LR + A[Cache Entry Expires] --> B{Pool cleanup?} + B -->|Before| C["❌ New untracked pool created β€” leaked"] + B -->|After| D["βœ… Pool closed on eviction"] +``` + --- ## New Providers and Endpoints @@ -438,6 +471,7 @@ The Compliance Playground lets you test any guardrail against our pre-built eval - Fix Redis connection pool reliability β€” prevent connection exhaustion under load - [PR #21717](https://github.com/BerriAI/litellm/pull/21717) - Fix Prisma connection self-heal for auth and runtime reconnection (reverted, will be re-introduced with fixes) - [PR #21706](https://github.com/BerriAI/litellm/pull/21706) +- Close streaming connections to prevent connection pool exhaustion - [PR #21213](https://github.com/BerriAI/litellm/pull/21213) - Make `PodLockManager.release_lock` atomic compare-and-delete - [PR #21226](https://github.com/BerriAI/litellm/pull/21226) --- diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index fdccc2174c..fa090b6ecf 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -166,6 +166,7 @@ const sidebars = { "tutorials/cursor_integration", "tutorials/github_copilot_integration", "tutorials/litellm_gemini_cli", + "tutorials/google_genai_sdk", "tutorials/litellm_qwen_code_cli", "tutorials/openai_codex" ] @@ -180,6 +181,7 @@ const sidebars = { slug: "/agent_sdks" }, items: [ + "tutorials/openai_agents_sdk", "tutorials/claude_agent_sdk", "tutorials/copilotkit_sdk", "tutorials/google_adk", @@ -419,6 +421,7 @@ const sidebars = { "proxy/dynamic_rate_limit", "proxy/rate_limit_tiers", "proxy/temporary_budget_increase", + "proxy/budget_reset_and_tz", ], }, "proxy/caching", diff --git a/enterprise/LICENSE.md b/enterprise/LICENSE.md index 5cd298ce65..c14a2a0c48 100644 --- a/enterprise/LICENSE.md +++ b/enterprise/LICENSE.md @@ -7,7 +7,7 @@ With regard to the BerriAI Software: This software and associated documentation files (the "Software") may only be used in production, if you (and any entity that you represent) have agreed to, and are in compliance with, the BerriAI Subscription Terms of Service, available -via [call](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) or email (info@berri.ai) (the "Enterprise Terms"), or other +via [call](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) or email (info@berri.ai) (the "Enterprise Terms"), or other agreement governing the use of the Software, as agreed by you and BerriAI, and otherwise have a valid BerriAI Enterprise license for the correct number of user seats. Subject to the foregoing sentence, you are free to diff --git a/enterprise/README.md b/enterprise/README.md index d5c27bab67..3b2ada6dd8 100644 --- a/enterprise/README.md +++ b/enterprise/README.md @@ -4,6 +4,6 @@ Code in this folder is licensed under a commercial license. Please review the [L **These features are covered under the LiteLLM Enterprise contract** -πŸ‘‰ **Using in an Enterprise / Need specific features ?** Meet with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat?month=2024-02) +πŸ‘‰ **Using in an Enterprise / Need specific features ?** Meet with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions?month=2024-02) See all Enterprise Features here πŸ‘‰ [Docs](https://docs.litellm.ai/docs/proxy/enterprise) diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index d3e0476930..2f2e444850 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -16,6 +16,10 @@ from litellm_enterprise.types.enterprise_callbacks.send_emails import ( from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache +from litellm.constants import ( + EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, + EMAIL_BUDGET_ALERT_TTL, +) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER from litellm.integrations.email_templates.key_created_email import ( @@ -24,14 +28,14 @@ from litellm.integrations.email_templates.key_created_email import ( from litellm.integrations.email_templates.key_rotated_email import ( KEY_ROTATED_EMAIL_TEMPLATE, ) -from litellm.integrations.email_templates.user_invitation_email import ( - USER_INVITATION_EMAIL_TEMPLATE, -) from litellm.integrations.email_templates.templates import ( MAX_BUDGET_ALERT_EMAIL_TEMPLATE, SOFT_BUDGET_ALERT_EMAIL_TEMPLATE, TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE, ) +from litellm.integrations.email_templates.user_invitation_email import ( + USER_INVITATION_EMAIL_TEMPLATE, +) from litellm.proxy._types import ( CallInfo, InvitationNew, @@ -41,10 +45,6 @@ from litellm.proxy._types import ( ) from litellm.secret_managers.main import get_secret_bool from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL -from litellm.constants import ( - EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, - EMAIL_BUDGET_ALERT_TTL, -) class BaseEmailLogger(CustomLogger): @@ -121,10 +121,16 @@ class BaseEmailLogger(CustomLogger): ) # Check if API key should be included in email - include_api_key = get_secret_bool(secret_name="EMAIL_INCLUDE_API_KEY", default_value=True) + include_api_key = get_secret_bool( + secret_name="EMAIL_INCLUDE_API_KEY", default_value=True + ) if include_api_key is None: include_api_key = True # Default to True if not set - key_token_display = send_key_created_email_event.virtual_key if include_api_key else "[Key hidden for security - retrieve from dashboard]" + key_token_display = ( + send_key_created_email_event.virtual_key + if include_api_key + else "[Key hidden for security - retrieve from dashboard]" + ) email_html_content = KEY_CREATED_EMAIL_TEMPLATE.format( email_logo_url=email_params.logo_url, @@ -162,10 +168,16 @@ class BaseEmailLogger(CustomLogger): ) # Check if API key should be included in email - include_api_key = get_secret_bool(secret_name="EMAIL_INCLUDE_API_KEY", default_value=True) + include_api_key = get_secret_bool( + secret_name="EMAIL_INCLUDE_API_KEY", default_value=True + ) if include_api_key is None: include_api_key = True # Default to True if not set - key_token_display = send_key_rotated_email_event.virtual_key if include_api_key else "[Key hidden for security - retrieve from dashboard]" + key_token_display = ( + send_key_rotated_email_event.virtual_key + if include_api_key + else "[Key hidden for security - retrieve from dashboard]" + ) email_html_content = KEY_ROTATED_EMAIL_TEMPLATE.format( email_logo_url=email_params.logo_url, @@ -201,7 +213,9 @@ class BaseEmailLogger(CustomLogger): ) # Format budget values - soft_budget_str = f"${event.soft_budget}" if event.soft_budget is not None else "N/A" + soft_budget_str = ( + f"${event.soft_budget}" if event.soft_budget is not None else "N/A" + ) spend_str = f"${event.spend}" if event.spend is not None else "$0.00" max_budget_info = "" if event.max_budget is not None: @@ -231,13 +245,13 @@ class BaseEmailLogger(CustomLogger): """ # Collect all recipient emails recipient_emails: List[str] = [] - + # Add additional alert emails from team metadata.soft_budget_alert_emails if hasattr(event, "alert_emails") and event.alert_emails: for email in event.alert_emails: if email and email not in recipient_emails: # Avoid duplicates recipient_emails.append(email) - + # If no recipients found, skip sending if not recipient_emails: verbose_proxy_logger.warning( @@ -268,7 +282,9 @@ class BaseEmailLogger(CustomLogger): ) # Format budget values - soft_budget_str = f"${event.soft_budget}" if event.soft_budget is not None else "N/A" + soft_budget_str = ( + f"${event.soft_budget}" if event.soft_budget is not None else "N/A" + ) spend_str = f"${event.spend}" if event.spend is not None else "$0.00" max_budget_info = "" if event.max_budget is not None: @@ -286,7 +302,7 @@ class BaseEmailLogger(CustomLogger): base_url=email_params.base_url, email_support_contact=email_params.support_contact, ) - + # Send email to all recipients await self.send_email( from_email=self.DEFAULT_LITELLM_EMAIL, @@ -313,11 +329,17 @@ class BaseEmailLogger(CustomLogger): # Format budget values spend_str = f"${event.spend}" if event.spend is not None else "$0.00" - max_budget_str = f"${event.max_budget}" if event.max_budget is not None else "N/A" - + max_budget_str = ( + f"${event.max_budget}" if event.max_budget is not None else "N/A" + ) + # Calculate percentage and alert threshold percentage = int(EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100) - alert_threshold_str = f"${event.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE:.2f}" if event.max_budget is not None else "N/A" + alert_threshold_str = ( + f"${event.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE:.2f}" + if event.max_budget is not None + else "N/A" + ) email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format( email_logo_url=email_params.logo_url, @@ -382,7 +404,10 @@ class BaseEmailLogger(CustomLogger): # For non-team alerts, require either max_budget or soft_budget if user_info.max_budget is None and user_info.soft_budget is None: return - if user_info.soft_budget is not None and user_info.spend >= user_info.soft_budget: + if ( + user_info.soft_budget is not None + and user_info.spend >= user_info.soft_budget + ): # Generate cache key based on event type and identifier # Use appropriate ID based on event_group to ensure unique cache keys per entity type if user_info.event_group == Litellm_EntityType.TEAM: @@ -395,7 +420,7 @@ class BaseEmailLogger(CustomLogger): # For KEY and other types, use token or user_id _id = user_info.token or user_info.user_id or "default_id" _cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}" - + # Check if we've already sent this alert result = await _cache.async_get_cache(key=_cache_key) if result is None: @@ -420,14 +445,14 @@ class BaseEmailLogger(CustomLogger): event_group=user_info.event_group, alert_emails=user_info.alert_emails, ) - + try: # Use team-specific function for team alerts, otherwise use standard function if user_info.event_group == Litellm_EntityType.TEAM: await self.send_team_soft_budget_alert_email(webhook_event) else: await self.send_soft_budget_alert_email(webhook_event) - + # Cache the alert to prevent duplicate sends await _cache.async_set_cache( key=_cache_key, @@ -444,20 +469,27 @@ class BaseEmailLogger(CustomLogger): # For max_budget_alert, check if we've already sent an alert if type == "max_budget_alert": if user_info.max_budget is not None and user_info.spend is not None: - alert_threshold = user_info.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE - + alert_threshold = ( + user_info.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE + ) + # Only alert if we've crossed the threshold but haven't exceeded max_budget yet - if user_info.spend >= alert_threshold and user_info.spend < user_info.max_budget: + if ( + user_info.spend >= alert_threshold + and user_info.spend < user_info.max_budget + ): # Generate cache key based on event type and identifier _id = user_info.token or user_info.user_id or "default_id" _cache_key = f"email_budget_alerts:max_budget_alert:{_id}" - + # Check if we've already sent this alert result = await _cache.async_get_cache(key=_cache_key) if result is None: # Calculate percentage - percentage = int(EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100) - + percentage = int( + EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100 + ) + # Create WebhookEvent for max budget alert event_message = f"Max Budget Alert - {percentage}% of Maximum Budget Reached" webhook_event = WebhookEvent( @@ -478,10 +510,10 @@ class BaseEmailLogger(CustomLogger): projected_spend=user_info.projected_spend, event_group=user_info.event_group, ) - + try: await self.send_max_budget_alert_email(webhook_event) - + # Cache the alert to prevent duplicate sends await _cache.async_set_cache( key=_cache_key, @@ -525,9 +557,14 @@ class BaseEmailLogger(CustomLogger): unused_custom_fields = [] # Function to safely get custom value or default - def get_custom_or_default(custom_value: Optional[str], default_value: str, field_name: str) -> str: - if custom_value is not None: # Only check premium if trying to use custom value + def get_custom_or_default( + custom_value: Optional[str], default_value: str, field_name: str + ) -> str: + if ( + custom_value is not None + ): # Only check premium if trying to use custom value from litellm.proxy.proxy_server import premium_user + if premium_user is not True: unused_custom_fields.append(field_name) return default_value @@ -536,38 +573,48 @@ class BaseEmailLogger(CustomLogger): # Get parameters, falling back to defaults if custom values aren't allowed logo_url = get_custom_or_default(custom_logo, LITELLM_LOGO_URL, "logo URL") - support_contact = get_custom_or_default(custom_support, self.DEFAULT_SUPPORT_EMAIL, "support contact") - base_url = os.getenv("PROXY_BASE_URL", "http://0.0.0.0:4000") # Not a premium feature - signature = get_custom_or_default(custom_signature, EMAIL_FOOTER, "email signature") + support_contact = get_custom_or_default( + custom_support, self.DEFAULT_SUPPORT_EMAIL, "support contact" + ) + base_url = os.getenv( + "PROXY_BASE_URL", "http://0.0.0.0:4000" + ) # Not a premium feature + signature = get_custom_or_default( + custom_signature, EMAIL_FOOTER, "email signature" + ) # Get custom subject template based on email event type if email_event == EmailEvent.new_user_invitation: subject_template = get_custom_or_default( custom_subject_invitation, self.DEFAULT_SUBJECT_TEMPLATES[EmailEvent.new_user_invitation], - "invitation subject template" + "invitation subject template", ) elif email_event == EmailEvent.virtual_key_created: subject_template = get_custom_or_default( custom_subject_key_created, self.DEFAULT_SUBJECT_TEMPLATES[EmailEvent.virtual_key_created], - "key created subject template" + "key created subject template", ) elif email_event == EmailEvent.virtual_key_rotated: custom_subject_key_rotated = os.getenv("EMAIL_SUBJECT_KEY_ROTATED", None) subject_template = get_custom_or_default( custom_subject_key_rotated, self.DEFAULT_SUBJECT_TEMPLATES[EmailEvent.virtual_key_rotated], - "key rotated subject template" + "key rotated subject template", ) else: subject_template = "LiteLLM: {event_message}" - subject = subject_template.format(event_message=event_message) if event_message else "LiteLLM Notification" + subject = ( + subject_template.format(event_message=event_message) + if event_message + else "LiteLLM Notification" + ) - recipient_email: Optional[ - str - ] = user_email or await self._lookup_user_email_from_db(user_id=user_id) + recipient_email: Optional[str] = ( + user_email or await self._lookup_user_email_from_db(user_id=user_id) + ) if recipient_email is None: raise ValueError( f"User email not found for user_id: {user_id}. User email is required to send email." @@ -585,11 +632,9 @@ class BaseEmailLogger(CustomLogger): warning_msg = ( f"Email sent with default values instead of custom values for: {fields_str}. " "This is an Enterprise feature. To use custom email fields, please upgrade to LiteLLM Enterprise. " - "Schedule a meeting here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" - ) - verbose_proxy_logger.warning( - f"{warning_msg}" + "Schedule a meeting here: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" ) + verbose_proxy_logger.warning(f"{warning_msg}") return EmailParams( logo_url=logo_url, @@ -636,44 +681,49 @@ class BaseEmailLogger(CustomLogger): if not user_id: verbose_proxy_logger.debug("No user_id provided for invitation link") return base_url - + if not await self._is_prisma_client_available(): return base_url - + # Wait for any concurrent invitation creation to complete await self._wait_for_invitation_creation() - + # Get or create invitation invitation = await self._get_or_create_invitation(user_id) if not invitation: - verbose_proxy_logger.warning(f"Failed to get/create invitation for user_id: {user_id}") + verbose_proxy_logger.warning( + f"Failed to get/create invitation for user_id: {user_id}" + ) return base_url - + return self._construct_invitation_link(invitation.id, base_url) async def _is_prisma_client_available(self) -> bool: """Check if Prisma client is available""" from litellm.proxy.proxy_server import prisma_client - + if prisma_client is None: - verbose_proxy_logger.debug("Prisma client not found. Unable to lookup invitation") + verbose_proxy_logger.debug( + "Prisma client not found. Unable to lookup invitation" + ) return False return True async def _wait_for_invitation_creation(self) -> None: """ Wait for any concurrent invitation creation to complete. - + The UI calls /invitation/new to generate the invitation link. We wait to ensure any pending invitation creation is completed. """ import asyncio + await asyncio.sleep(10) async def _get_or_create_invitation(self, user_id: str): """ Get existing invitation or create a new one for the user - + Returns: Invitation object with id attribute, or None if failed """ @@ -681,31 +731,41 @@ class BaseEmailLogger(CustomLogger): create_invitation_for_user, ) from litellm.proxy.proxy_server import prisma_client - + if prisma_client is None: - verbose_proxy_logger.error("Prisma client is None in _get_or_create_invitation") + verbose_proxy_logger.error( + "Prisma client is None in _get_or_create_invitation" + ) return None - + try: # Try to get existing invitation - existing_invitations = await prisma_client.db.litellm_invitationlink.find_many( - where={"user_id": user_id}, - order={"created_at": "desc"}, + existing_invitations = ( + await prisma_client.db.litellm_invitationlink.find_many( + where={"user_id": user_id}, + order={"created_at": "desc"}, + ) ) - + if existing_invitations and len(existing_invitations) > 0: - verbose_proxy_logger.debug(f"Found existing invitation for user_id: {user_id}") + verbose_proxy_logger.debug( + f"Found existing invitation for user_id: {user_id}" + ) return existing_invitations[0] - + # Create new invitation if none exists - verbose_proxy_logger.debug(f"Creating new invitation for user_id: {user_id}") + verbose_proxy_logger.debug( + f"Creating new invitation for user_id: {user_id}" + ) return await create_invitation_for_user( data=InvitationNew(user_id=user_id), user_api_key_dict=UserAPIKeyAuth(user_id=user_id), ) - + except Exception as e: - verbose_proxy_logger.error(f"Error getting/creating invitation for user_id {user_id}: {e}") + verbose_proxy_logger.error( + f"Error getting/creating invitation for user_id {user_id}: {e}" + ) return None def _construct_invitation_link(self, invitation_id: str, base_url: str) -> str: diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index bf8bc46f72..4dcabb9c58 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -13,6 +13,9 @@ if TYPE_CHECKING: from litellm.router import Router +CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" + + class CheckBatchCost: def __init__( self, @@ -27,6 +30,25 @@ class CheckBatchCost: self.prisma_client: PrismaClient = prisma_client self.llm_router: Router = llm_router + async def _get_user_info(self, batch_id, user_id) -> dict: + """ + Look up user email and key alias by user_id for enriching the S3 callback metadata. + Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None). + """ + try: + user_row = await self.prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_id} + ) + if user_row is None: + return {} + return { + "user_api_key_user_email": getattr(user_row, "user_email", None), + "user_api_key_alias": getattr(user_row, "user_alias", None), + } + except Exception as e: + verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}") + return {} + async def check_batch_cost(self): """ Check if the batch JOB has been tracked. @@ -48,10 +70,12 @@ class CheckBatchCost: get_model_id_from_unified_batch_id, ) + # Look for all batches that have not yet been processed by CheckBatchCost jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many( where={ - "status": {"in": ["validating", "in_progress", "finalizing"]}, "file_purpose": "batch", + "batch_processed" : False, + "status": {"not_in": ["failed", "expired", "cancelled"]} } ) completed_jobs = [] @@ -107,6 +131,21 @@ class CheckBatchCost: f"Batch ID: {batch_id} is complete, tracking cost and usage" ) + # aretrieve_batch is called with the raw provider batch ID, so response.id + # is the raw provider value (e.g. "batch_20260223-0518.234"). We need the + # unified base64 ID in the S3 log so downstream consumers can correlate it + # back to the batch they submitted via the proxy. + # + # CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and + # calls async_success_handler(result=response) directly. That handler calls + # _build_standard_logging_payload(response, ...) which reads response.id at + # that point β€” so setting response.id here is sufficient. + # + # The HTTP endpoint does this substitution via the managed files hook + # (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely, + # so we do it explicitly here. + response.id = job.unified_object_id + # This background job runs as default_user_id, so going through the HTTP endpoint # would trigger check_managed_file_id_access and get 403. Instead, extract the raw # provider file ID and call afile_content directly with deployment credentials. @@ -171,11 +210,21 @@ class CheckBatchCost: function_id=str(uuid.uuid4()), ) + creator_user_id = job.created_by + user_info = await self._get_user_info(batch_id, job.created_by) + logging_obj.update_environment_variables( litellm_params={ + # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks + "proxy_server_request": { + "headers": { + "user-agent": CHECK_BATCH_COST_USER_AGENT, + } + }, "metadata": { - "user_api_key_user_id": job.created_by or "default-user-id", - } + "user_api_key_user_id": creator_user_id, + **user_info, + }, }, optional_params={}, ) @@ -191,8 +240,7 @@ class CheckBatchCost: completed_jobs.append(job) if len(completed_jobs) > 0: - # mark the jobs as complete await self.prisma_client.db.litellm_managedobjecttable.update_many( where={"id": {"in": [job.id for job in completed_jobs]}}, - data={"status": "complete"}, + data={"batch_processed": True, "status": "complete"}, ) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bda20e2f74..4fa050a84a 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -1086,11 +1086,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self, file_id: str ) -> List[Dict[str, Any]]: """ - Find batches in non-terminal states that reference this file. - - Non-terminal states: validating, in_progress, finalizing - Terminal states: completed, complete, failed, expired, cancelled - + Find batches that reference this file and still need cost tracking. + Find batches that are in non-terminal state and have not yet been processed by CheckBatchCost. Args: file_id: The unified file ID to check @@ -1121,7 +1118,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): batches = await self.prisma_client.db.litellm_managedobjecttable.find_many( where={ "file_purpose": "batch", - "status": {"in": ["validating", "in_progress", "finalizing"]}, + "batch_processed": False, + "status": {"not_in": ["failed", "expired", "cancelled"]} }, take=MAX_MATCHES_TO_RETURN, order={"created_at": "desc"}, @@ -1205,7 +1203,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): error_message += ( f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. " - f"Alternatively, wait for all batches to complete processing." + f"Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)." ) raise HTTPException( diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214124140_baseline_diff/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214124140_baseline_diff/migration.sql deleted file mode 100644 index 2f725d8380..0000000000 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214124140_baseline_diff/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This is an empty migration. - diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql new file mode 100644 index 0000000000..dd95d9d84a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql @@ -0,0 +1,60 @@ +-- CreateTable +CREATE TABLE "LiteLLM_DailyGuardrailMetrics" ( + "guardrail_id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "requests_evaluated" BIGINT NOT NULL DEFAULT 0, + "passed_count" BIGINT NOT NULL DEFAULT 0, + "blocked_count" BIGINT NOT NULL DEFAULT 0, + "flagged_count" BIGINT NOT NULL DEFAULT 0, + "avg_score" DOUBLE PRECISION, + "avg_latency_ms" DOUBLE PRECISION, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LiteLLM_DailyGuardrailMetrics_pkey" PRIMARY KEY ("guardrail_id","date") +); + +-- CreateTable +CREATE TABLE "LiteLLM_DailyPolicyMetrics" ( + "policy_id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "requests_evaluated" BIGINT NOT NULL DEFAULT 0, + "passed_count" BIGINT NOT NULL DEFAULT 0, + "blocked_count" BIGINT NOT NULL DEFAULT 0, + "flagged_count" BIGINT NOT NULL DEFAULT 0, + "avg_score" DOUBLE PRECISION, + "avg_latency_ms" DOUBLE PRECISION, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LiteLLM_DailyPolicyMetrics_pkey" PRIMARY KEY ("policy_id","date") +); + +-- CreateTable +CREATE TABLE "LiteLLM_SpendLogGuardrailIndex" ( + "request_id" TEXT NOT NULL, + "guardrail_id" TEXT NOT NULL, + "policy_id" TEXT, + "start_time" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LiteLLM_SpendLogGuardrailIndex_pkey" PRIMARY KEY ("request_id","guardrail_id") +); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyGuardrailMetrics_date_idx" ON "LiteLLM_DailyGuardrailMetrics"("date"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyGuardrailMetrics_guardrail_id_idx" ON "LiteLLM_DailyGuardrailMetrics"("guardrail_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyPolicyMetrics_date_idx" ON "LiteLLM_DailyPolicyMetrics"("date"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyPolicyMetrics_policy_id_idx" ON "LiteLLM_DailyPolicyMetrics"("policy_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_SpendLogGuardrailIndex_guardrail_id_start_time_idx" ON "LiteLLM_SpendLogGuardrailIndex"("guardrail_id", "start_time"); + +-- CreateIndex +CREATE INDEX "LiteLLM_SpendLogGuardrailIndex_policy_id_start_time_idx" ON "LiteLLM_SpendLogGuardrailIndex"("policy_id", "start_time"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260222000000_add_batch_processed_to_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260222000000_add_batch_processed_to_managed_object_table/migration.sql new file mode 100644 index 0000000000..ac390d164d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260222000000_add_batch_processed_to_managed_object_table/migration.sql @@ -0,0 +1,3 @@ +-- Add batch_processed column to LiteLLM_ManagedObjectTable +-- Set to true by CheckBatchCost after cost has been computed for a completed batch +ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN "batch_processed" BOOLEAN NOT NULL DEFAULT false; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 5d2cad6da5..4af7484148 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -813,6 +813,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t file_object Json // Stores the OpenAIFileObject file_purpose String // either 'batch' or 'fine-tune' status String? // check if batch cost has been tracked + batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed created_at DateTime @default(now()) created_by String? updated_at DateTime @updatedAt @@ -866,6 +867,54 @@ model LiteLLM_GuardrailsTable { updated_at DateTime @updatedAt } +// Daily guardrail metrics for usage dashboard (one row per guardrail per day) +model LiteLLM_DailyGuardrailMetrics { + guardrail_id String // logical id; may not FK if guardrail from config + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([guardrail_id, date]) + @@index([date]) + @@index([guardrail_id]) +} + +// Daily policy metrics for usage dashboard (one row per policy per day) +model LiteLLM_DailyPolicyMetrics { + policy_id String + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([policy_id, date]) + @@index([date]) + @@index([policy_id]) +} + +// Index for fast "last N logs for guardrail/policy" from SpendLogs +model LiteLLM_SpendLogGuardrailIndex { + request_id String + guardrail_id String + policy_id String? // set when run as part of a policy pipeline + start_time DateTime + + @@id([request_id, guardrail_id]) + @@index([guardrail_id, start_time]) + @@index([policy_id, start_time]) +} + // Prompt table for storing prompt configurations model LiteLLM_PromptTable { id String @id @default(uuid()) diff --git a/litellm/__init__.py b/litellm/__init__.py index 97f36a9b00..1e74b5692e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -339,6 +339,10 @@ model_cost_map_url: str = os.getenv( "LITELLM_MODEL_COST_MAP_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json", ) +blog_posts_url: str = os.getenv( + "LITELLM_BLOG_POSTS_URL", + "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/blog_posts.json", +) anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", @@ -405,6 +409,7 @@ disable_aiohttp_trust_env: bool = ( force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) +network_mock: bool = False # When True, use mock transport β€” no real network calls ####### STOP SEQUENCE LIMIT ####### disable_stop_sequence_limit: bool = False # when True, stop sequence limit is disabled @@ -614,8 +619,9 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 -def add_known_models(): - for key, value in model_cost.items(): +def add_known_models(model_cost_map: Optional[Dict] = None): + _map = model_cost_map if model_cost_map is not None else model_cost + for key, value in _map.items(): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model( key ): diff --git a/litellm/blog_posts.json b/litellm/blog_posts.json new file mode 100644 index 0000000000..15340514bc --- /dev/null +++ b/litellm/blog_posts.json @@ -0,0 +1,10 @@ +{ + "posts": [ + { + "title": "Incident Report: SERVER_ROOT_PATH regression broke UI routing", + "description": "How a single line removal caused UI 404s for all deployments using SERVER_ROOT_PATH, and the tests we added to prevent it from happening again.", + "date": "2026-02-21", + "url": "https://docs.litellm.ai/blog/server-root-path-incident" + } + ] +} diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index a03bff6068..ad02d2ea89 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -108,6 +108,7 @@ class Cache: qdrant_collection_name: Optional[str] = None, qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002", + qdrant_semantic_cache_vector_size: Optional[int] = None, # GCP IAM authentication parameters gcp_service_account: Optional[str] = None, gcp_ssl_ca_certs: Optional[str] = None, @@ -207,6 +208,7 @@ class Cache: similarity_threshold=similarity_threshold, quantization_config=qdrant_quantization_config, embedding_model=qdrant_semantic_cache_embedding_model, + vector_size=qdrant_semantic_cache_vector_size, ) elif type == LiteLLMCacheType.LOCAL: self.cache = InMemoryCache() diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 0e77b5a6c2..181effa01d 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -31,6 +31,7 @@ class QdrantSemanticCache(BaseCache): quantization_config=None, embedding_model="text-embedding-ada-002", host_type=None, + vector_size=None, ): import os @@ -53,6 +54,7 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model + self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE headers = {} # check if defined as os.environ/ variable @@ -138,7 +140,7 @@ class QdrantSemanticCache(BaseCache): new_collection_status = self.sync_client.put( url=f"{self.qdrant_api_base}/collections/{self.collection_name}", json={ - "vectors": {"size": QDRANT_VECTOR_SIZE, "distance": "Cosine"}, + "vectors": {"size": self.vector_size, "distance": "Cosine"}, "quantization_config": quantization_params, }, headers=self.headers, diff --git a/litellm/constants.py b/litellm/constants.py index 89992b459c..ee79f2fa56 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -242,9 +242,13 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = ( REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer" MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) -MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000)) # Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)) +# Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger. +# Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire. +MAX_SIZE_IN_MEMORY_QUEUE = int( + os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)) +) MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int( os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000) ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 05c3da224e..cc0f818b0a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -480,6 +480,7 @@ def cost_per_token( # noqa: PLR0915 model=model_without_prefix, custom_llm_provider=custom_llm_provider, usage=usage_block, + service_tier=service_tier, ) elif custom_llm_provider == "anthropic": return anthropic_cost_per_token(model=model, usage=usage_block) @@ -500,7 +501,9 @@ def cost_per_token( # noqa: PLR0915 model=model, usage=usage_block, response_time_ms=response_time_ms ) elif custom_llm_provider == "gemini": - return gemini_cost_per_token(model=model, usage=usage_block) + return gemini_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier + ) elif custom_llm_provider == "deepseek": return deepseek_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "perplexity": @@ -704,6 +707,36 @@ def _get_response_model(completion_response: Any) -> Optional[str]: return None +_GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: dict = { + # ON_DEMAND_PRIORITY maps to "priority" β€” selects input_cost_per_token_priority, etc. + "ON_DEMAND_PRIORITY": "priority", + # FLEX / BATCH maps to "flex" β€” selects input_cost_per_token_flex, etc. + "FLEX": "flex", + "BATCH": "flex", + # ON_DEMAND is standard pricing β€” no service_tier suffix applied + "ON_DEMAND": None, +} + + +def _map_traffic_type_to_service_tier(traffic_type: Optional[str]) -> Optional[str]: + """ + Map a Gemini usageMetadata.trafficType value to a LiteLLM service_tier string. + + This allows the same `_priority` / `_flex` cost-key suffix logic used for + OpenAI/Azure to work for Gemini and Vertex AI models. + + trafficType values seen in practice + ------------------------------------ + ON_DEMAND -> standard pricing (service_tier = None) + ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority") + FLEX / BATCH -> batch/flex pricing (service_tier = "flex") + """ + if traffic_type is None: + return None + service_tier = _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER.get(traffic_type.upper()) + return service_tier + + def _get_usage_object( completion_response: Any, ) -> Optional[Usage]: @@ -1145,6 +1178,20 @@ def completion_cost( # noqa: PLR0915 "custom_llm_provider", custom_llm_provider or None ) region_name = hidden_params.get("region_name", region_name) + + # For Gemini/Vertex AI responses, trafficType is stored in + # provider_specific_fields. Map it to the service_tier used + # by the cost key lookup (_priority / _flex suffixes) so that + # ON_DEMAND_PRIORITY requests are billed at priority prices. + if service_tier is None: + provider_specific = ( + hidden_params.get("provider_specific_fields") or {} + ) + raw_traffic_type = provider_specific.get("traffic_type") + if raw_traffic_type: + service_tier = _map_traffic_type_to_service_tier( + raw_traffic_type + ) else: if model is None: raise ValueError( diff --git a/litellm/integrations/SlackAlerting/hanging_request_check.py b/litellm/integrations/SlackAlerting/hanging_request_check.py index 713e790ba9..d2f70c9caf 100644 --- a/litellm/integrations/SlackAlerting/hanging_request_check.py +++ b/litellm/integrations/SlackAlerting/hanging_request_check.py @@ -172,4 +172,6 @@ Team Alias: `{hanging_request_data.team_alias}`""" level="Medium", alert_type=AlertType.llm_requests_hanging, alerting_metadata=hanging_request_data.alerting_metadata or {}, + request_model=hanging_request_data.model, + api_base=hanging_request_data.api_base, ) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index a525856db8..35634d5067 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -70,6 +70,7 @@ class SlackAlerting(CustomBatchLogger): ] = None, # if user wants to separate alerts to diff channels alerting_args={}, default_webhook_url: Optional[str] = None, + alert_type_config: Optional[Dict[str, dict]] = None, **kwargs, ): if alerting_threshold is None: @@ -92,6 +93,12 @@ class SlackAlerting(CustomBatchLogger): self.hanging_request_check = AlertingHangingRequestCheck( slack_alerting_object=self, ) + self.alert_type_config: Dict[str, AlertTypeConfig] = {} + if alert_type_config: + for key, val in alert_type_config.items(): + self.alert_type_config[key] = AlertTypeConfig(**val) if isinstance(val, dict) else val + self.digest_buckets: Dict[str, DigestEntry] = {} + self.digest_lock = asyncio.Lock() super().__init__(**kwargs, flush_lock=self.flush_lock) def update_values( @@ -102,6 +109,7 @@ class SlackAlerting(CustomBatchLogger): alert_to_webhook_url: Optional[Dict[AlertType, Union[List[str], str]]] = None, alerting_args: Optional[Dict] = None, llm_router: Optional[Router] = None, + alert_type_config: Optional[Dict[str, dict]] = None, ): if alerting is not None: self.alerting = alerting @@ -116,6 +124,9 @@ class SlackAlerting(CustomBatchLogger): if not self.periodic_started: asyncio.create_task(self.periodic_flush()) self.periodic_started = True + if alert_type_config is not None: + for key, val in alert_type_config.items(): + self.alert_type_config[key] = AlertTypeConfig(**val) if isinstance(val, dict) else val if alert_to_webhook_url is not None: # update the dict @@ -284,6 +295,8 @@ class SlackAlerting(CustomBatchLogger): level="Low", alert_type=AlertType.llm_too_slow, alerting_metadata=alerting_metadata, + request_model=model, + api_base=api_base, ) async def async_update_daily_reports( @@ -1354,13 +1367,15 @@ Model Info: return False - async def send_alert( + async def send_alert( # noqa: PLR0915 self, message: str, level: Literal["Low", "Medium", "High"], alert_type: AlertType, alerting_metadata: dict, user_info: Optional[WebhookEvent] = None, + request_model: Optional[str] = None, + api_base: Optional[str] = None, **kwargs, ): """ @@ -1376,6 +1391,8 @@ Model Info: Parameters: level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'. message: str - what is the alert about + request_model: Optional[str] - model name for digest grouping + api_base: Optional[str] - api base for digest grouping """ if self.alerting is None: return @@ -1413,6 +1430,44 @@ Model Info: from datetime import datetime + # Check if digest mode is enabled for this alert type + alert_type_name_str = getattr(alert_type, "value", str(alert_type)) + _atc = self.alert_type_config.get(alert_type_name_str) + if _atc is not None and _atc.digest: + # Resolve webhook URL for this alert type (needed for digest entry) + if ( + self.alert_to_webhook_url is not None + and alert_type in self.alert_to_webhook_url + ): + _digest_webhook: Optional[Union[str, List[str]]] = self.alert_to_webhook_url[alert_type] + elif self.default_webhook_url is not None: + _digest_webhook = self.default_webhook_url + else: + _digest_webhook = os.getenv("SLACK_WEBHOOK_URL", None) + if _digest_webhook is None: + raise ValueError("Missing SLACK_WEBHOOK_URL from environment") + + digest_key = f"{alert_type_name_str}:{request_model or ''}:{api_base or ''}" + + async with self.digest_lock: + now = datetime.now() + if digest_key in self.digest_buckets: + self.digest_buckets[digest_key]["count"] += 1 + self.digest_buckets[digest_key]["last_time"] = now + else: + self.digest_buckets[digest_key] = DigestEntry( + alert_type=alert_type_name_str, + request_model=request_model or "", + api_base=api_base or "", + first_message=message, + level=level, + count=1, + start_time=now, + last_time=now, + webhook_url=_digest_webhook, + ) + return # Suppress immediate alert; will be emitted by _flush_digest_buckets + # Get the current timestamp current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) @@ -1488,6 +1543,72 @@ Model Info: await asyncio.gather(*tasks) self.log_queue.clear() + async def _flush_digest_buckets(self): + """Flush any digest buckets whose interval has expired. + + For each expired bucket, formats a digest summary message and + appends it to the log_queue for delivery via the normal batching path. + """ + from datetime import datetime + + now = datetime.now() + flushed_keys: List[str] = [] + + async with self.digest_lock: + for key, entry in self.digest_buckets.items(): + alert_type_name = entry["alert_type"] + _atc = self.alert_type_config.get(alert_type_name) + if _atc is None: + continue + elapsed = (now - entry["start_time"]).total_seconds() + if elapsed < _atc.digest_interval: + continue + + # Build digest summary message + start_ts = entry["start_time"].strftime("%H:%M:%S") + end_ts = entry["last_time"].strftime("%H:%M:%S") + start_date = entry["start_time"].strftime("%Y-%m-%d") + end_date = entry["last_time"].strftime("%Y-%m-%d") + formatted_message = ( + f"Alert type: `{alert_type_name}` (Digest)\n" + f"Level: `{entry['level']}`\n" + f"Start: `{start_date} {start_ts}`\n" + f"End: `{end_date} {end_ts}`\n" + f"Count: `{entry['count']}`\n\n" + f"Message: {entry['first_message']}" + ) + _proxy_base_url = os.getenv("PROXY_BASE_URL", None) + if _proxy_base_url is not None: + formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`" + + payload = {"text": formatted_message} + headers = {"Content-type": "application/json"} + webhook_url = entry["webhook_url"] + + if isinstance(webhook_url, list): + for url in webhook_url: + self.log_queue.append( + {"url": url, "headers": headers, "payload": payload, "alert_type": alert_type_name} + ) + else: + self.log_queue.append( + {"url": webhook_url, "headers": headers, "payload": payload, "alert_type": alert_type_name} + ) + flushed_keys.append(key) + + for key in flushed_keys: + del self.digest_buckets[key] + + async def periodic_flush(self): + """Override base periodic_flush to also flush digest buckets.""" + while True: + await asyncio.sleep(self.flush_interval) + try: + await self._flush_digest_buckets() + except Exception as e: + verbose_proxy_logger.debug(f"Error flushing digest buckets: {str(e)}") + await self.flush_queue() + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): """Log deployment latency""" try: diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 4a1e3e41e9..bf330944ef 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -587,9 +587,10 @@ class CustomGuardrail(CustomLogger): elif "litellm_metadata" in request_data: _append_guardrail_info(request_data["litellm_metadata"]) else: - verbose_logger.warning( - "unable to log guardrail information. No metadata found in request_data" - ) + # Ensure guardrail info is always logged (e.g. proxy may not have set + # metadata yet). Attach to "metadata" so spend log / standard logging see it. + request_data["metadata"] = {} + _append_guardrail_info(request_data["metadata"]) async def apply_guardrail( self, diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 9a317cfcf0..70c28c4e06 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -8,8 +8,9 @@ duration_in_seconds is used in diff parts of the code base, example import re import time -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta, timezone, tzinfo from typing import Optional, Tuple +from zoneinfo import ZoneInfo def _extract_from_regex(duration: str) -> Tuple[int, str]: @@ -116,7 +117,7 @@ def get_next_standardized_reset_time( - Next reset time at a standardized interval in the specified timezone """ # Set up timezone and normalize current time - current_time, timezone = _setup_timezone(current_time, timezone_str) + current_time, tz = _setup_timezone(current_time, timezone_str) # Parse duration value, unit = _parse_duration(duration) @@ -131,7 +132,7 @@ def get_next_standardized_reset_time( # Handle different time units if unit == "d": - return _handle_day_reset(current_time, base_midnight, value, timezone) + return _handle_day_reset(current_time, base_midnight, value, tz) elif unit == "h": return _handle_hour_reset(current_time, base_midnight, value) elif unit == "m": @@ -147,22 +148,13 @@ def get_next_standardized_reset_time( def _setup_timezone( current_time: datetime, timezone_str: str = "UTC" -) -> Tuple[datetime, timezone]: +) -> Tuple[datetime, tzinfo]: """Set up timezone and normalize current time to that timezone.""" try: if timezone_str is None: - tz = timezone.utc + tz: tzinfo = timezone.utc else: - # Map common timezone strings to their UTC offsets - timezone_map = { - "US/Eastern": timezone(timedelta(hours=-4)), # EDT - "US/Pacific": timezone(timedelta(hours=-7)), # PDT - "Asia/Kolkata": timezone(timedelta(hours=5, minutes=30)), # IST - "Asia/Bangkok": timezone(timedelta(hours=7)), # ICT (Indochina Time) - "Europe/London": timezone(timedelta(hours=1)), # BST - "UTC": timezone.utc, - } - tz = timezone_map.get(timezone_str, timezone.utc) + tz = ZoneInfo(timezone_str) except Exception: # If timezone is invalid, fall back to UTC tz = timezone.utc @@ -190,7 +182,7 @@ def _parse_duration(duration: str) -> Tuple[Optional[int], Optional[str]]: def _handle_day_reset( - current_time: datetime, base_midnight: datetime, value: int, timezone: timezone + current_time: datetime, base_midnight: datetime, value: int, tz: tzinfo ) -> datetime: """Handle day-based reset times.""" # Handle zero value - immediate expiration @@ -215,7 +207,7 @@ def _handle_day_reset( minute=0, second=0, microsecond=0, - tzinfo=timezone, + tzinfo=tz, ) else: next_reset = datetime( @@ -226,7 +218,7 @@ def _handle_day_reset( minute=0, second=0, microsecond=0, - tzinfo=timezone, + tzinfo=tz, ) return next_reset else: # Custom day value - next interval is value days from current diff --git a/litellm/litellm_core_utils/get_blog_posts.py b/litellm/litellm_core_utils/get_blog_posts.py new file mode 100644 index 0000000000..4f054c78ff --- /dev/null +++ b/litellm/litellm_core_utils/get_blog_posts.py @@ -0,0 +1,128 @@ +""" +Pulls the latest LiteLLM blog posts from GitHub. + +Falls back to the bundled local backup on any failure. +GitHub JSON URL is configured via litellm.blog_posts_url (or LITELLM_BLOG_POSTS_URL env var). + +Disable remote fetching entirely: + export LITELLM_LOCAL_BLOG_POSTS=True +""" + +import json +import os +import time +from importlib.resources import files +from typing import Any, Dict, List, Optional + +import httpx +from pydantic import BaseModel + +from litellm import verbose_logger + +BLOG_POSTS_TTL_SECONDS: int = 3600 # 1 hour + + +class BlogPost(BaseModel): + title: str + description: str + date: str + url: str + + +class BlogPostsResponse(BaseModel): + posts: List[BlogPost] + + +class GetBlogPosts: + """ + Fetches, validates, and caches LiteLLM blog posts. + + Mirrors the structure of GetModelCostMap: + - Fetches from GitHub with a 5-second timeout + - Validates the response has a non-empty ``posts`` list + - Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour) + - Falls back to the bundled local backup on any failure + """ + + _cached_posts: Optional[List[Dict[str, str]]] = None + _last_fetch_time: float = 0.0 + + @staticmethod + def load_local_blog_posts() -> List[Dict[str, str]]: + """Load the bundled local backup blog posts.""" + content = json.loads( + files("litellm") + .joinpath("blog_posts.json") + .read_text(encoding="utf-8") + ) + return content.get("posts", []) + + @staticmethod + def fetch_remote_blog_posts(url: str, timeout: int = 5) -> dict: + """ + Fetch blog posts JSON from a remote URL. + + Returns the parsed response. Raises on network/parse errors. + """ + response = httpx.get(url, timeout=timeout) + response.raise_for_status() + return response.json() + + @staticmethod + def validate_blog_posts(data: Any) -> bool: + """Return True if data is a dict with a non-empty ``posts`` list.""" + if not isinstance(data, dict): + verbose_logger.warning( + "LiteLLM: Blog posts response is not a dict (type=%s). " + "Falling back to local backup.", + type(data).__name__, + ) + return False + posts = data.get("posts") + if not isinstance(posts, list) or len(posts) == 0: + verbose_logger.warning( + "LiteLLM: Blog posts response has no valid 'posts' list. " + "Falling back to local backup.", + ) + return False + return True + + @classmethod + def get_blog_posts(cls, url: str) -> List[Dict[str, str]]: + """ + Return the blog posts list. + + Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS. + Fetches from ``url`` otherwise, falling back to local backup on failure. + """ + if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": + return cls.load_local_blog_posts() + + now = time.time() + cached = cls._cached_posts + if cached is not None and (now - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS: + return cached + + try: + data = cls.fetch_remote_blog_posts(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: Failed to fetch blog posts from %s: %s. " + "Falling back to local backup.", + url, + str(e), + ) + return cls.load_local_blog_posts() + + if not cls.validate_blog_posts(data): + return cls.load_local_blog_posts() + + posts = data["posts"] + cls._cached_posts = posts + cls._last_fetch_time = now + return posts + + +def get_blog_posts(url: str) -> List[Dict[str, str]]: + """Public entry point β€” returns the blog posts list.""" + return GetBlogPosts.get_blog_posts(url=url) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index e622a31745..f9398979f9 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -11,6 +11,7 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True import json import os from importlib.resources import files +from typing import Optional import httpx @@ -151,6 +152,37 @@ class GetModelCostMap: return response.json() +class ModelCostMapSourceInfo: + """Tracks the source of the currently loaded model cost map.""" + + source: str = "local" # "local" or "remote" + url: Optional[str] = None + is_env_forced: bool = False + fallback_reason: Optional[str] = None + + +# Module-level singleton tracking the source of the current cost map +_cost_map_source_info = ModelCostMapSourceInfo() + + +def get_model_cost_map_source_info() -> dict: + """ + Return metadata about where the current model cost map was loaded from. + + Returns a dict with: + - source: "local" or "remote" + - url: the remote URL attempted (or None for local-only) + - is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage + - fallback_reason: human-readable reason if remote failed and local was used + """ + return { + "source": _cost_map_source_info.source, + "url": _cost_map_source_info.url, + "is_env_forced": _cost_map_source_info.is_env_forced, + "fallback_reason": _cost_map_source_info.fallback_reason, + } + + def get_model_cost_map(url: str) -> dict: """ Public entry point β€” returns the model cost map dict. @@ -166,8 +198,15 @@ def get_model_cost_map(url: str) -> dict: # Note: can't use get_secret_bool here β€” this runs during litellm.__init__ # before litellm._key_management_settings is set. if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.source = "local" + _cost_map_source_info.url = None + _cost_map_source_info.is_env_forced = True + _cost_map_source_info.fallback_reason = None return GetModelCostMap.load_local_model_cost_map() + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + try: content = GetModelCostMap.fetch_remote_model_cost_map(url) except Exception as e: @@ -177,6 +216,8 @@ def get_model_cost_map(url: str) -> dict: url, str(e), ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {str(e)}" return GetModelCostMap.load_local_model_cost_map() # Validate using cached count (cheap int comparison, no file I/O) @@ -189,6 +230,10 @@ def get_model_cost_map(url: str) -> dict: "Using local backup instead. url=%s", url, ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" return GetModelCostMap.load_local_model_cost_map() + _cost_map_source_info.source = "remote" + _cost_map_source_info.fallback_reason = None return content diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3648dc691e..0601e7e845 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4709,6 +4709,7 @@ class StandardLoggingPayloadSetup: custom_pricing: Optional[bool], custom_llm_provider: Optional[str], init_response_obj: Union[Any, BaseModel, dict], + api_base: Optional[str] = None, ) -> StandardLoggingModelInformation: model_cost_name = _select_model_name_for_cost_calc( model=None, @@ -4723,7 +4724,9 @@ class StandardLoggingPayloadSetup: else: try: _model_cost_information = litellm.get_model_info( - model=model_cost_name, custom_llm_provider=custom_llm_provider + model=model_cost_name, + custom_llm_provider=custom_llm_provider, + api_base=api_base, ) model_cost_information = StandardLoggingModelInformation( model_map_key=model_cost_name, @@ -5236,6 +5239,7 @@ def get_standard_logging_object_payload( custom_pricing=custom_pricing, custom_llm_provider=kwargs.get("custom_llm_provider"), init_response_obj=init_response_obj, + api_base=litellm_params.get("api_base"), ) response_cost: float = kwargs.get("response_cost", 0) or 0.0 diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 7c41e1bbe6..a9fd0f4ea8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -200,8 +200,14 @@ def _get_token_base_cost( ## CHECK IF ABOVE THRESHOLD # Optimization: collect threshold keys first to avoid sorting all model_info keys. # Most models don't have threshold pricing, so we can return early. + # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority) + # so that the threshold detection loop only processes standard keys. The + # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") + k + for k in model_info + if k.startswith("input_cost_per_token_above_") + and not any(k.endswith(f"_{st.value}") for st in ServiceTier) ] if not threshold_keys: return ( @@ -224,14 +230,34 @@ def _get_token_base_cost( 1000 if "k" in threshold_str else 1 ) if usage.prompt_tokens > threshold: + # Prefer a service_tier-specific above-threshold key when available, + # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini + # ON_DEMAND_PRIORITY. Falls back to the standard key automatically + # via _get_cost_per_unit's service_tier fallback logic. + tiered_input_key = ( + _get_service_tier_cost_key( + f"input_cost_per_token_above_{threshold_str}_tokens", + service_tier, + ) + if service_tier + else key + ) prompt_base_cost = cast( - float, _get_cost_per_unit(model_info, key, prompt_base_cost) + float, _get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost) + ) + tiered_output_key = ( + _get_service_tier_cost_key( + f"output_cost_per_token_above_{threshold_str}_tokens", + service_tier, + ) + if service_tier + else f"output_cost_per_token_above_{threshold_str}_tokens" ) completion_base_cost = cast( float, _get_cost_per_unit( model_info, - f"output_cost_per_token_above_{threshold_str}_tokens", + tiered_output_key, completion_base_cost, ), ) @@ -517,6 +543,7 @@ def _calculate_input_cost( cache_read_cost: float, cache_creation_cost: float, cache_creation_cost_above_1hr: float, + service_tier: Optional[str] = None, ) -> float: """ Calculates the input cost for a given model, prompt tokens, and completion tokens. @@ -528,8 +555,11 @@ def _calculate_input_cost( ### AUDIO COST if prompt_tokens_details["audio_tokens"]: + audio_cost_key = _get_service_tier_cost_key( + "input_cost_per_audio_token", service_tier + ) prompt_cost += calculate_cost_component( - model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"] + model_info, audio_cost_key, prompt_tokens_details["audio_tokens"] ) ### IMAGE TOKEN COST @@ -659,6 +689,7 @@ def generic_cost_per_token( # noqa: PLR0915 cache_read_cost=cache_read_cost, cache_creation_cost=cache_creation_cost, cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + service_tier=service_tier, ) ## CALCULATE OUTPUT COST diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 1f739a60b4..ccd5c1dd8f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -6,7 +6,17 @@ import logging import threading import time import traceback -from typing import Any, Callable, Dict, List, Optional, Union, cast +from typing import ( + Any, + AsyncIterator, + Callable, + Dict, + Iterator, + List, + Optional, + Union, + cast, +) import anyio import httpx @@ -151,10 +161,10 @@ class CustomStreamWrapper: self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: Optional[int] = None - def __iter__(self): + def __iter__(self) -> Iterator["ModelResponseStream"]: return self - def __aiter__(self): + def __aiter__(self) -> AsyncIterator["ModelResponseStream"]: return self async def aclose(self): @@ -1726,7 +1736,7 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason = "tool_calls" return model_response - def __next__(self): # noqa: PLR0915 + def __next__(self) -> "ModelResponseStream": # noqa: PLR0915 cache_hit = False if ( self.custom_llm_provider is not None @@ -1748,7 +1758,7 @@ class CustomStreamWrapper: chunk = next(self.completion_stream) if chunk is not None and chunk != b"": print_verbose( - f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}; custom_llm_provider: {self.custom_llm_provider}" + f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk.decode('utf-8', errors='replace') if isinstance(chunk, bytes) else chunk}; custom_llm_provider: {self.custom_llm_provider}" ) response: Optional[ModelResponseStream] = self.chunk_creator( chunk=chunk @@ -1900,7 +1910,7 @@ class CustomStreamWrapper: return self.completion_stream - async def __anext__(self): # noqa: PLR0915 + async def __anext__(self) -> "ModelResponseStream": # noqa: PLR0915 cache_hit = False if ( self.custom_llm_provider is not None @@ -1996,9 +2006,7 @@ class CustomStreamWrapper: else: chunk = next(self.completion_stream) if chunk is not None and chunk != b"": - processed_chunk: Optional[ - ModelResponseStream - ] = self.chunk_creator(chunk=chunk) + processed_chunk = self.chunk_creator(chunk=chunk) if processed_chunk is None: continue diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 271406f2f7..cf9b18c464 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -5,10 +5,50 @@ Helper util for handling anthropic-specific cost calculation from typing import TYPE_CHECKING, Optional, Tuple -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_token_base_cost, + _parse_prompt_tokens_details, + calculate_cache_writing_cost, + generic_cost_per_token, +) if TYPE_CHECKING: from litellm.types.utils import ModelInfo, Usage +import litellm + + +def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: + """ + Return only the cache-related portion of the prompt cost (cache read + cache write). + + These costs must NOT be scaled by geo/speed multipliers because the old + explicit ``fast/`` model entries carried unchanged cache rates while + multiplying only the regular input/output token costs. + """ + if usage.prompt_tokens_details is None: + return 0.0 + + prompt_tokens_details = _parse_prompt_tokens_details(usage) + _, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = ( + _get_token_base_cost(model_info=model_info, usage=usage) + ) + + cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost + + if ( + prompt_tokens_details["cache_creation_tokens"] + or prompt_tokens_details["cache_creation_token_details"] is not None + ): + cache_cost += calculate_cache_writing_cost( + cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], + cache_creation_token_details=prompt_tokens_details[ + "cache_creation_token_details" + ], + cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost=cache_creation_cost, + ) + + return cache_cost def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: @@ -22,20 +62,34 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - model_with_prefix = model - - # First, prepend inference_geo if present - if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]: - model_with_prefix = f"{usage.inference_geo}/{model_with_prefix}" - - # Then, prepend speed if it's "fast" - if hasattr(usage, "speed") and usage.speed == "fast": - model_with_prefix = f"fast/{model_with_prefix}" - prompt_cost, completion_cost = generic_cost_per_token( - model=model_with_prefix, usage=usage, custom_llm_provider="anthropic" + model=model, usage=usage, custom_llm_provider="anthropic" ) + # Apply provider_specific_entry multipliers for geo/speed routing + try: + model_info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + provider_specific_entry: dict = model_info.get("provider_specific_entry") or {} + + multiplier = 1.0 + if ( + hasattr(usage, "inference_geo") + and usage.inference_geo + and usage.inference_geo.lower() not in ["global", "not_available"] + ): + multiplier *= provider_specific_entry.get( + usage.inference_geo.lower(), 1.0 + ) + if hasattr(usage, "speed") and usage.speed == "fast": + multiplier *= provider_specific_entry.get("fast", 1.0) + + if multiplier != 1.0: + cache_cost = _compute_cache_only_cost(model_info=model_info, usage=usage) + prompt_cost = (prompt_cost - cache_cost) * multiplier + cache_cost + completion_cost *= multiplier + except Exception: + pass + return prompt_cost, completion_cost diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 50cada42b8..1ad91a43df 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -118,10 +118,11 @@ class BaseVideoConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, + variant: Optional[str] = None, ) -> Tuple[str, Dict]: """ Transform the video content request into a URL and data/params - + Returns: Tuple[str, Dict]: (url, params) for the video content request """ diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index fdcbe1a824..e29b07ca3a 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -202,52 +202,84 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): return optional_params + # Providers whose InvokeModel body uses the Converse API format + # (messages + inferenceConfig + image blocks). Nova is the primary + # example; add others here as they adopt the same schema. + CONVERSE_INVOKE_PROVIDERS = ("nova",) + def _map_openai_to_bedrock_params( self, openai_request_body: Dict[str, Any], provider: Optional[str] = None, ) -> Dict[str, Any]: """ - Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic + Transform OpenAI request body to Bedrock-compatible modelInput + parameters using existing transformation logic. + + Routes to the correct per-provider transformation so that the + resulting dict matches the InvokeModel body that Bedrock expects + for batch inference. """ from litellm.types.utils import LlmProviders + _model = openai_request_body.get("model", "") messages = openai_request_body.get("messages", []) - - # Use existing Anthropic transformation logic for Anthropic models + optional_params = { + k: v + for k, v in openai_request_body.items() + if k not in ["model", "messages"] + } + + # --- Anthropic: use existing AmazonAnthropicClaudeConfig --- if provider == LlmProviders.ANTHROPIC: from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) - - anthropic_config = AmazonAnthropicClaudeConfig() - - # Extract optional params (everything except model and messages) - optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]} - mapped_params = anthropic_config.map_openai_params( + + config = AmazonAnthropicClaudeConfig() + mapped_params = config.map_openai_params( non_default_params={}, optional_params=optional_params, model=_model, - drop_params=False + drop_params=False, ) - - # Transform using existing Anthropic logic - bedrock_params = anthropic_config.transform_request( + return config.transform_request( model=_model, messages=messages, optional_params=mapped_params, litellm_params={}, - headers={} + headers={}, ) - return bedrock_params - else: - # For other providers, use basic mapping - bedrock_params = { - "messages": messages, - **{k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]} - } - return bedrock_params + # --- Converse API providers (e.g. Nova): use AmazonConverseConfig + # to correctly convert image_url blocks to Bedrock image format + # and wrap inference params inside inferenceConfig. --- + if provider in self.CONVERSE_INVOKE_PROVIDERS: + from litellm.llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig, + ) + + converse_config = AmazonConverseConfig() + mapped_params = converse_config.map_openai_params( + non_default_params=optional_params, + optional_params={}, + model=_model, + drop_params=False, + ) + return converse_config.transform_request( + model=_model, + messages=messages, + optional_params=mapped_params, + litellm_params={}, + headers={}, + ) + + # --- All other providers: passthrough (OpenAI-compatible models + # like openai.gpt-oss-*, qwen, deepseek, etc.) --- + return { + "messages": messages, + **optional_params, + } def _transform_openai_jsonl_content_to_bedrock_jsonl_content( self, openai_jsonl_content: List[Dict[str, Any]] diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0a5364bfcf..7267532933 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3014,8 +3014,11 @@ class BaseLLMHTTPHandler: raise ValueError(f"Unsupported transformed_request type: {type(transformed_request)}") # Store the upload URL in litellm_params for the transformation method + # Honour the URL already set by transform_create_file_request (e.g. Bedrock pre-signed S3 uploads), + # fall back to api_base for providers that do not set it. litellm_params_with_url = dict(litellm_params) - litellm_params_with_url["upload_url"] = api_base + if "upload_url" not in litellm_params: + litellm_params_with_url["upload_url"] = api_base return provider_config.transform_create_file_response( model=None, @@ -5397,6 +5400,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + variant: Optional[str] = None, ) -> Union[bytes, Coroutine[Any, Any, bytes]]: """ Handle video content download requests. @@ -5412,6 +5416,7 @@ class BaseLLMHTTPHandler: extra_headers=extra_headers, api_key=api_key, client=client, + variant=variant, ) if client is None or not isinstance(client, HTTPHandler): @@ -5443,6 +5448,7 @@ class BaseLLMHTTPHandler: api_base=api_base, litellm_params=litellm_params, headers=headers, + variant=variant, ) try: @@ -5485,6 +5491,7 @@ class BaseLLMHTTPHandler: extra_headers: Optional[Dict[str, Any]] = None, api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + variant: Optional[str] = None, ) -> bytes: """ Async version of the video content download handler. @@ -5519,6 +5526,7 @@ class BaseLLMHTTPHandler: api_base=api_base, litellm_params=litellm_params, headers=headers, + variant=variant, ) try: @@ -5594,7 +5602,7 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = video_remix_provider_config.validate_environment( - api_key=api_key, + api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", ) @@ -5676,7 +5684,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = video_remix_provider_config.validate_environment( - api_key=api_key, + api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", ) diff --git a/litellm/llms/custom_httpx/mock_transport.py b/litellm/llms/custom_httpx/mock_transport.py new file mode 100644 index 0000000000..262d0dff12 --- /dev/null +++ b/litellm/llms/custom_httpx/mock_transport.py @@ -0,0 +1,92 @@ +""" +Mock httpx transport that returns valid OpenAI ChatCompletion responses. + +Activated via `litellm_settings: { network_mock: true }`. +Intercepts at the httpx transport layer β€” the lowest point before bytes hit the wire β€” +so the full proxy -> router -> OpenAI SDK -> httpx path is exercised. +""" + +import json +import time +import uuid +from typing import Tuple + +import httpx + + +# --------------------------------------------------------------------------- +# Pre-built response templates +# --------------------------------------------------------------------------- + +def _mock_id() -> str: + return f"chatcmpl-mock-{uuid.uuid4().hex[:8]}" + + +def _chat_completion_json(model: str) -> dict: + """Return a minimal valid ChatCompletion object.""" + return { + "id": _mock_id(), + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Mock response", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + } + + +# --------------------------------------------------------------------------- +# Transport +# --------------------------------------------------------------------------- + +_JSON_HEADERS = { + "content-type": "application/json", +} + + +class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): + """ + httpx transport that returns canned OpenAI ChatCompletion responses. + + Supports both async (AsyncOpenAI) and sync (OpenAI) SDK paths. + """ + + @staticmethod + def _parse_request(request: httpx.Request) -> Tuple[str, bool]: + """Extract model from the request body.""" + try: + body = json.loads(request.content) + except (json.JSONDecodeError, ValueError): + return ("mock-model", False) + model = body.get("model", "mock-model") + return (model, False) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + model, _ = self._parse_request(request) + body = json.dumps(_chat_completion_json(model)).encode() + return httpx.Response( + status_code=200, + headers=_JSON_HEADERS, + content=body, + ) + + def handle_request(self, request: httpx.Request) -> httpx.Response: + model, _ = self._parse_request(request) + body = json.dumps(_chat_completion_json(model)).encode() + return httpx.Response( + status_code=200, + headers=_JSON_HEADERS, + content=body, + ) diff --git a/litellm/llms/gemini/cost_calculator.py b/litellm/llms/gemini/cost_calculator.py index 471421b487..79242fe01d 100644 --- a/litellm/llms/gemini/cost_calculator.py +++ b/litellm/llms/gemini/cost_calculator.py @@ -4,13 +4,15 @@ This file is used to calculate the cost of the Gemini API. Handles the context caching for Gemini API. """ -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING, Optional, Tuple if TYPE_CHECKING: from litellm.types.utils import ModelInfo, Usage -def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: +def cost_per_token( + model: str, usage: "Usage", service_tier: Optional[str] = None +) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -19,7 +21,7 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token return generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="gemini" + model=model, usage=usage, custom_llm_provider="gemini", service_tier=service_tier ) diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 4120d1cad2..7daeb75b65 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -393,10 +393,11 @@ class GeminiVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, + variant: Optional[str] = None, ) -> Tuple[str, Dict]: """ Transform the video content request for Veo API. - + For Veo, we need to: 1. Get operation status to extract video URI 2. Return download URL for the video diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index c4d08c83a2..ed14b6a331 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, from httpx._models import Headers, Response import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -223,7 +223,9 @@ class OllamaConfig(BaseConfig): or get_secret_str("OLLAMA_API_KEY") ) - def get_model_info(self, model: str) -> ModelInfoBase: + def get_model_info( + self, model: str, api_base: Optional[str] = None + ) -> ModelInfoBase: """ curl http://localhost:11434/api/show -d '{ "name": "mistral" @@ -231,7 +233,11 @@ class OllamaConfig(BaseConfig): """ if model.startswith("ollama/") or model.startswith("ollama_chat/"): model = model.split("/", 1)[1] - api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" + api_base = ( + api_base + or get_secret_str("OLLAMA_API_BASE") + or "http://localhost:11434" + ) api_key = self.get_api_key() headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} @@ -242,8 +248,21 @@ class OllamaConfig(BaseConfig): headers=headers, ) except Exception as e: - raise Exception( - f"OllamaError: Error getting model info for {model}. Set Ollama API Base via `OLLAMA_API_BASE` environment variable. Error: {e}" + verbose_logger.debug( + "OllamaError: Could not get model info for %s from %s. Error: %s", + model, + api_base, + e, + ) + return ModelInfoBase( + key=model, + litellm_provider="ollama", + mode="chat", + input_cost_per_token=0.0, + output_cost_per_token=0.0, + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, ) model_info = response.json() diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 28de9f1303..61f150f1c2 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -205,6 +205,11 @@ class BaseOpenAILLM: if litellm.aclient_session is not None: return litellm.aclient_session + if getattr(litellm, "network_mock", False): + from litellm.llms.custom_httpx.mock_transport import MockOpenAITransport + + return httpx.AsyncClient(transport=MockOpenAITransport()) + # Get unified SSL configuration ssl_config = get_ssl_configuration() @@ -225,6 +230,11 @@ class BaseOpenAILLM: if litellm.client_session is not None: return litellm.client_session + if getattr(litellm, "network_mock", False): + from litellm.llms.custom_httpx.mock_transport import MockOpenAITransport + + return httpx.Client(transport=MockOpenAITransport()) + # Get unified SSL configuration ssl_config = get_ssl_configuration() diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 0dd7940a92..5c880ab665 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -172,18 +172,22 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, + variant: Optional[str] = None, ) -> Tuple[str, Dict]: """ Transform the video content request for OpenAI API. - + OpenAI API expects the following request: - GET /v1/videos/{video_id}/content + - GET /v1/videos/{video_id}/content?variant=thumbnail """ original_video_id = extract_original_video_id(video_id) - + # Construct the URL for video content download url = f"{api_base.rstrip('/')}/{original_video_id}/content" - + if variant is not None: + url = f"{url}?variant={variant}" + # No additional data needed for GET content request data: Dict[str, Any] = {} diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 5a46ebb664..318a732dc2 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -310,10 +310,11 @@ class RunwayMLVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, + variant: Optional[str] = None, ) -> Tuple[str, Dict]: """ Transform the video content request for RunwayML API. - + RunwayML doesn't have a separate content download endpoint. The video URL is returned in the task output field. We'll retrieve the task and extract the video URL. diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index e98dc75915..e7ac453e94 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -224,6 +224,7 @@ def cost_per_token( model: str, custom_llm_provider: str, usage: Usage, + service_tier: Optional[str] = None, ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -233,6 +234,8 @@ def cost_per_token( - custom_llm_provider: str, either "vertex_ai-*" or "gemini" - prompt_tokens: float, the number of input tokens - completion_tokens: float, the number of output tokens + - service_tier: optional tier derived from Gemini trafficType + ("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch). Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -266,4 +269,5 @@ def cost_per_token( model=model, custom_llm_provider=custom_llm_provider, usage=usage, + service_tier=service_tier, ) diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index 66cd143764..8cdccc4cd6 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -455,6 +455,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, + variant: Optional[str] = None, ) -> Tuple[str, Dict]: """ Transform the video content request for Veo API. diff --git a/litellm/main.py b/litellm/main.py index 52e7475169..8b239c454f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -147,6 +147,7 @@ from litellm.utils import ( token_counter, validate_and_fix_openai_messages, validate_and_fix_openai_tools, + validate_and_fix_thinking_param, validate_chat_completion_tool_choice, validate_openai_optional_params, ) @@ -1103,6 +1104,8 @@ def completion( # type: ignore # noqa: PLR0915 tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) # validate optional params stop = validate_openai_optional_params(stop=stop) + # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens) + thinking = validate_and_fix_thinking_param(thinking=thinking) ######### unpacking kwargs ##################### args = locals() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e54eaf89d7..4f4e99f099 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8295,37 +8295,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "us/claude-sonnet-4-6": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, - "cache_read_input_token_cost": 3.3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, - "input_cost_per_token": 3.3e-06, - "input_cost_per_token_above_200k_tokens": 6.6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_200k_tokens": 2.475e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "inference_geo": "us" - }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -8517,100 +8486,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "us/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/us/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -8641,69 +8521,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/claude-opus-4-6-20260205": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "us/claude-opus-4-6-20260205": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -14768,7 +14590,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14819,7 +14648,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -14919,7 +14755,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -14963,7 +14806,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -15014,7 +14862,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "vertex_ai/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -15065,7 +14920,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 1.25e-07, @@ -16860,6 +16722,8 @@ "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_priority": 1.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 2.5e-06, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -16873,8 +16737,11 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, + "output_cost_per_token_priority": 1e-05, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-05, "rpm": 2000, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_service_tier": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -16979,7 +16846,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -17027,7 +16901,12 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -17078,7 +16957,14 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -17129,7 +17015,14 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -17175,7 +17068,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "gemini/gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 0.0, @@ -37749,4 +37647,4 @@ "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." } } -} +} \ No newline at end of file diff --git a/litellm/policy_templates_backup.json b/litellm/policy_templates_backup.json index be2352866b..5c93ec11d4 100644 --- a/litellm/policy_templates_backup.json +++ b/litellm/policy_templates_backup.json @@ -2454,5 +2454,367 @@ "Injection Protection" ], "estimated_latency_ms": 1 + }, + { + "id": "pdpa-singapore", + "title": "Singapore PDPA \u2014 Personal Data Protection", + "description": "Singapore Personal Data Protection Act (PDPA) compliance. Covers 5 obligation areas: personal identifier collection (s.13 Consent), sensitive data profiling (Advisory Guidelines), Do Not Call Registry violations (Part IX), overseas data transfers (s.26), and automated profiling without human oversight (Model AI Governance Framework). Also includes regex-based PII detection for NRIC/FIN, Singapore phone numbers, postal codes, passports, UEN, and bank account numbers. Zero-cost keyword-based detection.", + "icon": "ShieldCheckIcon", + "iconColor": "text-red-500", + "iconBg": "bg-red-50", + "guardrails": [ + "pdpa-sg-pii-identifiers", + "pdpa-sg-contact-information", + "pdpa-sg-financial-data", + "pdpa-sg-business-identifiers", + "pdpa-sg-personal-identifiers", + "pdpa-sg-sensitive-data", + "pdpa-sg-do-not-call", + "pdpa-sg-data-transfer", + "pdpa-sg-profiling-automated-decisions" + ], + "complexity": "High", + "guardrailDefinitions": [ + { + "guardrail_name": "pdpa-sg-pii-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_nric", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "passport_singapore", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore NRIC/FIN and passport numbers for PDPA compliance" + } + }, + { + "guardrail_name": "pdpa-sg-contact-information", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_phone", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "sg_postal_code", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "email", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore phone numbers, postal codes, and email addresses" + } + }, + { + "guardrail_name": "pdpa-sg-financial-data", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_bank_account", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "credit_card", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore bank account numbers and credit card numbers" + } + }, + { + "guardrail_name": "pdpa-sg-business-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_uen", + "action": "MASK" + } + ], + "pattern_redaction_format": "[UEN_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore Unique Entity Numbers (business registration)" + } + }, + { + "guardrail_name": "pdpa-sg-personal-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_personal_identifiers", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_personal_identifiers.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA s.13 \u2014 Blocks unauthorized collection, harvesting, or extraction of Singapore personal identifiers (NRIC/FIN, SingPass, passports)" + } + }, + { + "guardrail_name": "pdpa-sg-sensitive-data", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_sensitive_data", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_sensitive_data.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA Advisory Guidelines \u2014 Blocks profiling or inference of sensitive personal data categories (race, religion, health, politics) for Singapore residents" + } + }, + { + "guardrail_name": "pdpa-sg-do-not-call", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_do_not_call", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_do_not_call.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA Part IX \u2014 Blocks generation of unsolicited marketing lists and DNC Registry bypass attempts for Singapore phone numbers" + } + }, + { + "guardrail_name": "pdpa-sg-data-transfer", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_data_transfer", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_data_transfer.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA s.26 \u2014 Blocks unprotected overseas transfer of Singapore personal data without adequate safeguards" + } + }, + { + "guardrail_name": "pdpa-sg-profiling-automated-decisions", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_profiling_automated_decisions", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_profiling_automated_decisions.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA + Model AI Governance Framework \u2014 Blocks automated profiling and decision-making about Singapore residents without human oversight" + } + } + ], + "templateData": { + "policy_name": "pdpa-singapore", + "description": "Singapore PDPA compliance policy. Covers personal identifier protection (s.13), sensitive data profiling (Advisory Guidelines), Do Not Call Registry (Part IX), overseas data transfers (s.26), and automated profiling (Model AI Governance Framework). Includes regex-based PII detection for NRIC/FIN, phone numbers, postal codes, passports, UEN, and bank accounts.", + "guardrails_add": [ + "pdpa-sg-pii-identifiers", + "pdpa-sg-contact-information", + "pdpa-sg-financial-data", + "pdpa-sg-business-identifiers", + "pdpa-sg-personal-identifiers", + "pdpa-sg-sensitive-data", + "pdpa-sg-do-not-call", + "pdpa-sg-data-transfer", + "pdpa-sg-profiling-automated-decisions" + ], + "guardrails_remove": [] + }, + "tags": [ + "PII Protection", + "Regulatory", + "Singapore" + ], + "estimated_latency_ms": 1 + }, + { + "id": "mas-ai-risk-management", + "title": "Singapore MAS \u2014 AI Risk Management for Financial Institutions", + "description": "Monetary Authority of Singapore (MAS) AI Risk Management for Financial Institutions alignment. Covers 5 enforceable obligation areas: fairness & bias in financial decisions, transparency & explainability of AI models, human oversight for consequential actions, data governance for financial customer data, and model security against adversarial attacks. Based on Guidelines on Artificial Intelligence Risk Management (MAS), and aligned with the 2018 FEAT Principles and Project MindForge. Zero-cost keyword-based detection.", + "icon": "ShieldCheckIcon", + "iconColor": "text-blue-600", + "iconBg": "bg-blue-50", + "guardrails": [ + "mas-sg-fairness-bias", + "mas-sg-transparency-explainability", + "mas-sg-human-oversight", + "mas-sg-data-governance", + "mas-sg-model-security" + ], + "complexity": "High", + "guardrailDefinitions": [ + { + "guardrail_name": "mas-sg-fairness-bias", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_fairness_bias", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_fairness_bias.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks discriminatory AI practices in financial services that score, deny, or price based on protected attributes (race, religion, age, gender, nationality)" + } + }, + { + "guardrail_name": "mas-sg-transparency-explainability", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_transparency_explainability", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks deployment of opaque or unexplainable AI systems for consequential financial decisions" + } + }, + { + "guardrail_name": "mas-sg-human-oversight", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_human_oversight", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_human_oversight.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks fully automated financial AI decisions without human-in-the-loop for consequential actions (loans, claims, trading)" + } + }, + { + "guardrail_name": "mas-sg-data-governance", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_data_governance", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_data_governance.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks unauthorized sharing, exposure, or mishandling of financial customer data without proper governance and data lineage" + } + }, + { + "guardrail_name": "mas-sg-model-security", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_model_security", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_model_security.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks adversarial attacks, model poisoning, inversion, and exfiltration attempts targeting financial AI systems" + } + } + ], + "templateData": { + "policy_name": "mas-ai-risk-management", + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) for Financial Institutions alignment. Covers fairness & bias, transparency & explainability, human oversight, data governance, and model security. Aligned with the 2018 FEAT Principles, Project MindForge, and NIST AI RMF.", + "guardrails_add": [ + "mas-sg-fairness-bias", + "mas-sg-transparency-explainability", + "mas-sg-human-oversight", + "mas-sg-data-governance", + "mas-sg-model-security" + ], + "guardrails_remove": [] + }, + "tags": [ + "Financial Services", + "Regulatory", + "Singapore" + ], + "estimated_latency_ms": 1 } ] diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 813a4fb3a6..6b84d90a32 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -23,11 +23,6 @@ model_list: guardrails: - - guardrail_name: mcp-user-permissions - litellm_params: - guardrail: mcp_end_user_permission - mode: pre_call - default_on: true - guardrail_name: "airline-competitor-intent" guardrail_id: "airline-competitor-intent" litellm_params: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 12f7fe2c3c..40fae4e4a5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import time import traceback from datetime import datetime from typing import ( @@ -441,10 +442,18 @@ class ProxyBaseLLMRequestProcessing: ), **( { - "x-litellm-timing-pre-processing-ms": str(hidden_params.get("timing_pre_processing_ms", None)), - "x-litellm-timing-llm-api-ms": str(hidden_params.get("timing_llm_api_ms", None)), - "x-litellm-timing-post-processing-ms": str(hidden_params.get("timing_post_processing_ms", None)), - "x-litellm-timing-message-copy-ms": str(hidden_params.get("timing_message_copy_ms", None)), + "x-litellm-timing-pre-processing-ms": str( + hidden_params.get("timing_pre_processing_ms", None) + ), + "x-litellm-timing-llm-api-ms": str( + hidden_params.get("timing_llm_api_ms", None) + ), + "x-litellm-timing-post-processing-ms": str( + hidden_params.get("timing_post_processing_ms", None) + ), + "x-litellm-timing-message-copy-ms": str( + hidden_params.get("timing_message_copy_ms", None) + ), } if LITELLM_DETAILED_TIMING else {} @@ -564,16 +573,6 @@ class ProxyBaseLLMRequestProcessing: ) -> Tuple[dict, LiteLLMLoggingObj]: start_time = datetime.now() # start before calling guardrail hooks - # Calculate request queue time if arrival_time is available - # Use start_time.timestamp() to avoid extra time.time() call for better performance - proxy_server_request = self.data.get("proxy_server_request", {}) - arrival_time = proxy_server_request.get("arrival_time") - queue_time_seconds = None - if arrival_time is not None: - # Convert start_time (datetime) to timestamp for calculation - processing_start_time = start_time.timestamp() - queue_time_seconds = processing_start_time - arrival_time - self.data = await add_litellm_data_to_request( data=self.data, request=request, @@ -583,6 +582,15 @@ class ProxyBaseLLMRequestProcessing: proxy_config=proxy_config, ) + # Calculate request queue time after add_litellm_data_to_request + # which sets arrival_time in proxy_server_request + proxy_server_request = self.data.get("proxy_server_request", {}) + arrival_time = proxy_server_request.get("arrival_time") + queue_time_seconds = None + if arrival_time is not None: + processing_start_time = time.time() + queue_time_seconds = processing_start_time - arrival_time + # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name @@ -634,7 +642,7 @@ class ProxyBaseLLMRequestProcessing: self.data["litellm_call_id"] = request.headers.get( "x-litellm-call-id", str(uuid.uuid4()) ) - + ### AUTO STREAM USAGE TRACKING ### # If always_include_stream_usage is enabled and this is a streaming request # automatically add stream_options={'include_usage': True} if not already set @@ -650,7 +658,7 @@ class ProxyBaseLLMRequestProcessing: and "include_usage" not in self.data["stream_options"] ): self.data["stream_options"]["include_usage"] = True - + ### CALL HOOKS ### - modify/reject incoming data before calling the model ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call @@ -710,7 +718,9 @@ class ProxyBaseLLMRequestProcessing: "Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s", len(_payload_str), MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, - list(self.data.keys()) if isinstance(self.data, dict) else type(self.data).__name__, + list(self.data.keys()) + if isinstance(self.data, dict) + else type(self.data).__name__, ) else: verbose_proxy_logger.debug( @@ -913,9 +923,9 @@ class ProxyBaseLLMRequestProcessing: # aliasing/routing, but the OpenAI-compatible response `model` field should reflect # what the client sent. if requested_model_from_client: - self.data["_litellm_client_requested_model"] = ( - requested_model_from_client - ) + self.data[ + "_litellm_client_requested_model" + ] = requested_model_from_client if route_type == "allm_passthrough_route": # Check if response is an async generator if self._is_streaming_response(response): @@ -1510,9 +1520,9 @@ class ProxyBaseLLMRequestProcessing: # Add cache-related fields to **params (handled by Usage.__init__) if cache_creation_input_tokens is not None: - usage_kwargs["cache_creation_input_tokens"] = ( - cache_creation_input_tokens - ) + usage_kwargs[ + "cache_creation_input_tokens" + ] = cache_creation_input_tokens if cache_read_input_tokens is not None: usage_kwargs["cache_read_input_tokens"] = cache_read_input_tokens diff --git a/litellm/proxy/common_utils/timezone_utils.py b/litellm/proxy/common_utils/timezone_utils.py index a289e5328b..700a9197f6 100644 --- a/litellm/proxy/common_utils/timezone_utils.py +++ b/litellm/proxy/common_utils/timezone_utils.py @@ -1,27 +1,23 @@ from datetime import datetime, timezone +import litellm from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time def get_budget_reset_timezone(): """ - Get the budget reset timezone from general_settings. + Get the budget reset timezone from litellm_settings. Falls back to UTC if not specified. + + litellm_settings values are set as attributes on the litellm module + by proxy_server.py at startup (via setattr(litellm, key, value)). """ - # Import at function level to avoid circular imports - from litellm.proxy.proxy_server import general_settings - - if general_settings: - litellm_settings = general_settings.get("litellm_settings", {}) - if litellm_settings and "timezone" in litellm_settings: - return litellm_settings["timezone"] - - return "UTC" + return getattr(litellm, "timezone", None) or "UTC" def get_budget_reset_time(budget_duration: str): """ - Get the budget reset time from general_settings. + Get the budget reset time based on the configured timezone. Falls back to UTC if not specified. """ diff --git a/litellm/proxy/db/db_transaction_queue/base_update_queue.py b/litellm/proxy/db/db_transaction_queue/base_update_queue.py index a5ec1c3eaf..e37200c02e 100644 --- a/litellm/proxy/db/db_transaction_queue/base_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/base_update_queue.py @@ -23,6 +23,15 @@ class BaseUpdateQueue: def __init__(self): self.update_queue = asyncio.Queue(maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE) self.MAX_SIZE_IN_MEMORY_QUEUE = MAX_SIZE_IN_MEMORY_QUEUE + if MAX_SIZE_IN_MEMORY_QUEUE >= LITELLM_ASYNCIO_QUEUE_MAXSIZE: + verbose_proxy_logger.warning( + "Misconfigured queue thresholds: MAX_SIZE_IN_MEMORY_QUEUE (%d) >= LITELLM_ASYNCIO_QUEUE_MAXSIZE (%d). " + "The spend aggregation check will never trigger because the asyncio.Queue blocks at %d items. " + "Set MAX_SIZE_IN_MEMORY_QUEUE to a value less than LITELLM_ASYNCIO_QUEUE_MAXSIZE (recommended: 80%% of it).", + MAX_SIZE_IN_MEMORY_QUEUE, + LITELLM_ASYNCIO_QUEUE_MAXSIZE, + LITELLM_ASYNCIO_QUEUE_MAXSIZE, + ) async def add_update(self, update): """Enqueue an update.""" diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index e14782fa1f..c083c60cb4 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -14,21 +14,27 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry -from litellm.types.guardrails import (PII_ENTITY_CATEGORIES_MAP, - ApplyGuardrailRequest, - ApplyGuardrailResponse, - BaseLitellmParams, - BedrockGuardrailConfigModel, Guardrail, - GuardrailEventHooks, - GuardrailInfoResponse, - GuardrailUIAddGuardrailSettings, - LakeraV2GuardrailConfigModel, - ListGuardrailsResponse, LitellmParams, - PatchGuardrailRequest, PiiAction, - PiiEntityType, - PresidioPresidioConfigModelUserInterface, - SupportedGuardrailIntegrations, - ToolPermissionGuardrailConfigModel) +from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router +from litellm.types.guardrails import ( + PII_ENTITY_CATEGORIES_MAP, + ApplyGuardrailRequest, + ApplyGuardrailResponse, + BaseLitellmParams, + BedrockGuardrailConfigModel, + Guardrail, + GuardrailEventHooks, + GuardrailInfoResponse, + GuardrailUIAddGuardrailSettings, + LakeraV2GuardrailConfigModel, + ListGuardrailsResponse, + LitellmParams, + PatchGuardrailRequest, + PiiAction, + PiiEntityType, + PresidioPresidioConfigModelUserInterface, + SupportedGuardrailIntegrations, + ToolPermissionGuardrailConfigModel, +) #### GUARDRAILS ENDPOINTS #### @@ -147,8 +153,7 @@ async def list_guardrails_v2(): ``` """ from litellm.litellm_core_utils.litellm_logging import _get_masked_values - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -288,8 +293,7 @@ async def create_guardrail(request: CreateGuardrailRequest): } ``` """ - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -378,8 +382,7 @@ async def update_guardrail(guardrail_id: str, request: UpdateGuardrailRequest): } ``` """ - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -447,8 +450,7 @@ async def delete_guardrail(guardrail_id: str): } ``` """ - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -541,8 +543,7 @@ async def patch_guardrail(guardrail_id: str, request: PatchGuardrailRequest): } ``` """ - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -664,8 +665,7 @@ async def get_guardrail_info(guardrail_id: str): """ from litellm.litellm_core_utils.litellm_logging import _get_masked_values - from litellm.proxy.guardrails.guardrail_registry import \ - IN_MEMORY_GUARDRAIL_HANDLER + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER from litellm.proxy.proxy_server import prisma_client from litellm.types.guardrails import GUARDRAIL_DEFINITION_LOCATION @@ -740,8 +740,10 @@ async def get_guardrail_ui_settings(): - Content filter settings (patterns and categories) """ from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( - PATTERN_CATEGORIES, get_available_content_categories, - get_pattern_metadata) + PATTERN_CATEGORIES, + get_available_content_categories, + get_pattern_metadata, + ) # Convert the PII_ENTITY_CATEGORIES_MAP to the format expected by the UI category_maps = [] @@ -1313,8 +1315,7 @@ async def get_provider_specific_params(): } ### get the config model for the guardrail - go through the registry and get the config model for the guardrail - from litellm.proxy.guardrails.guardrail_registry import \ - guardrail_class_registry + from litellm.proxy.guardrails.guardrail_registry import guardrail_class_registry for guardrail_name, guardrail_class in guardrail_class_registry.items(): guardrail_config_model = guardrail_class.get_config_model() @@ -1442,8 +1443,9 @@ async def test_custom_code_guardrail(request: TestCustomCodeGuardrailRequest): import concurrent.futures import re - from litellm.proxy.guardrails.guardrail_hooks.custom_code.primitives import \ - get_custom_code_primitives + from litellm.proxy.guardrails.guardrail_hooks.custom_code.primitives import ( + get_custom_code_primitives, + ) # Security validation patterns FORBIDDEN_PATTERNS = [ @@ -1633,3 +1635,7 @@ async def apply_guardrail( ) except Exception as e: raise handle_exception_on_proxy(e) + + +# Usage (dashboard) endpoints: overview, detail, logs +router.include_router(guardrails_usage_router) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json index 88328de09b..934ccbe2ff 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -493,6 +493,56 @@ "description": "Detects airline flight numbers (major IATA 2-letter codes + 1-4 digit flight number) when near flight context", "keyword_pattern": "\\b(?:flight|departure|arrival|gate|boarding|schedule|operate|route|aircraft|plane|outbound|inbound|leg|sector|flying)\\b", "allow_word_numbers": false + }, + { + "name": "sg_nric", + "display_name": "NRIC/FIN (Singapore National ID)", + "pattern": "\\b[STFGM]\\d{7}[A-Z]\\b", + "category": "Singapore PII Patterns", + "description": "Detects Singapore NRIC and FIN numbers (S/T for citizens/PRs, F/G/M for foreigners + 7 digits + checksum letter)" + }, + { + "name": "sg_phone", + "display_name": "Phone Number (Singapore)", + "pattern": "(? None: + self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + + self.api_key = api_key or os.environ.get("NOMA_API_KEY") + self.api_base = (api_base or os.environ.get("NOMA_API_BASE") or _DEFAULT_API_BASE).rstrip("/") + self.application_id = application_id or os.environ.get("NOMA_APPLICATION_ID") + if monitor_mode is None: + self.monitor_mode = os.environ.get("NOMA_MONITOR_MODE", "false").lower() == "true" + else: + self.monitor_mode = monitor_mode + + if block_failures is None: + self.block_failures = os.environ.get("NOMA_BLOCK_FAILURES", "true").lower() == "true" + else: + self.block_failures = block_failures + + if self._requires_api_key(api_base=self.api_base) and not self.api_key: + raise ValueError("Noma v2 guardrail requires api_key when using Noma SaaS endpoint") + + if "supported_event_hooks" not in kwargs: + kwargs["supported_event_hooks"] = [ + GuardrailEventHooks.pre_call, + GuardrailEventHooks.during_call, + GuardrailEventHooks.post_call, + GuardrailEventHooks.pre_mcp_call, + GuardrailEventHooks.during_mcp_call, + ] + + super().__init__(**kwargs) + + @staticmethod + def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: + from litellm.types.proxy.guardrails.guardrail_hooks.noma import ( + NomaV2GuardrailConfigModel, + ) + + return NomaV2GuardrailConfigModel + + def _get_authorization_header(self) -> str: + if not self.api_key: + return "" + return f"Bearer {self.api_key}" + + @staticmethod + def _requires_api_key(api_base: str) -> bool: + parsed = urlparse(api_base) + return parsed.hostname == _DEFAULT_API_BASE_HOSTNAME + + @staticmethod + def _get_non_empty_str(value: Any) -> Optional[str]: + if not isinstance(value, str): + return None + stripped = value.strip() + return stripped or None + + def _resolve_action_from_response( + self, + response_json: dict, + ) -> _Action: + action = response_json.get("action") + if isinstance(action, str): + try: + return _Action(action) + except ValueError: + pass + + raise ValueError("Noma v2 response missing valid action") + + def _build_scan_payload( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"], + application_id: Optional[str], + ) -> dict: + payload_request_data = deepcopy(request_data) + if logging_obj is not None: + payload_request_data["litellm_logging_obj"] = getattr(logging_obj, "model_call_details", None) + + payload: dict[str, Any] = { + "inputs": inputs, + "request_data": payload_request_data, + "input_type": input_type, + "monitor_mode": self.monitor_mode, + } + if application_id: + payload["application_id"] = application_id + return payload + + @staticmethod + def _sanitize_payload_for_transport(payload: dict) -> dict: + def _default(obj: Any) -> Any: + if hasattr(obj, "model_dump"): + try: + return obj.model_dump() + except Exception: + pass + return str(obj) + + try: + json_str = json.dumps(payload, default=_default) + except (ValueError, TypeError): + json_str = safe_dumps(payload) + + safe_payload = safe_json_loads(json_str, default={}) + if safe_payload == {} and payload: + verbose_proxy_logger.warning( + "Noma v2 guardrail: payload serialization failed, falling back to empty payload" + ) + + if isinstance(safe_payload, dict): + return safe_payload + + verbose_proxy_logger.warning( + "Noma v2 guardrail: payload sanitization produced non-dict output (type=%s), falling back to empty payload", + type(safe_payload).__name__, + ) + return {} + + async def _call_noma_scan( + self, + payload: dict, + ) -> dict: + headers: dict[str, str] = {"Content-Type": "application/json"} + authorization_header = self._get_authorization_header() + if authorization_header: + headers["Authorization"] = authorization_header + + endpoint = f"{self.api_base}{_AIDR_SCAN_ENDPOINT}" + sanitized_payload = self._sanitize_payload_for_transport(payload) + response = await self.async_handler.post( + url=endpoint, + headers=headers, + json=sanitized_payload, + ) + verbose_proxy_logger.debug( + "Noma v2 AIDR response: status_code=%s body=%s", + response.status_code, + response.text, + ) + response.raise_for_status() + response_json = response.json() + verbose_proxy_logger.debug( + "Noma v2 AIDR response parsed: %s", + json.dumps(response_json, default=str), + ) + return response_json + + def _add_guardrail_observability( + self, + request_data: dict, + start_time: datetime, + guardrail_status: GuardrailStatus, + guardrail_json_response: Any, + ) -> None: + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_provider="noma_v2", + guardrail_json_response=guardrail_json_response, + request_data=request_data, + guardrail_status=guardrail_status, + start_time=start_time.timestamp(), + end_time=end_time.timestamp(), + duration=duration, + ) + + def _apply_action( + self, + inputs: GenericGuardrailAPIInputs, + response_json: dict, + action: _Action, + ) -> GenericGuardrailAPIInputs: + if action == _Action.BLOCKED: + raise NomaBlockedMessage(response_json) + + if action == _Action.GUARDRAIL_INTERVENED: + updated_inputs = cast(GenericGuardrailAPIInputs, dict(inputs)) + for field in _INTERVENED_INPUT_FIELDS: + value = response_json.get(field) + if isinstance(value, list): + updated_inputs[field] = value # type: ignore[literal-required] + return updated_inputs + + return inputs + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + start_time = datetime.now() + guardrail_status: GuardrailStatus = "success" + guardrail_json_response: Any = {} + dynamic_params = self.get_guardrail_dynamic_request_body_params(request_data) + if not isinstance(dynamic_params, dict): + dynamic_params = {} + response_json: Optional[dict] = None + + # Per-request dynamic params can override configured application context. + application_id = self._get_non_empty_str(dynamic_params.get("application_id")) + + if application_id is None: + application_id = self._get_non_empty_str(self.application_id) + + try: + payload = self._build_scan_payload( + inputs=inputs, + request_data=request_data, + input_type=input_type, + logging_obj=logging_obj, + application_id=application_id, + ) + + response_json = await self._call_noma_scan(payload=payload) + if self.monitor_mode: + action = _Action.NONE + else: + action = self._resolve_action_from_response(response_json=response_json) + guardrail_json_response = response_json + verbose_proxy_logger.debug( + "Noma v2 guardrail decision: input_type=%s action=%s", + input_type, + action.value, + ) + processed_inputs = self._apply_action( + inputs=inputs, + response_json=response_json, + action=action, + ) + + guardrail_status = "success" if action == _Action.NONE else "guardrail_intervened" + return processed_inputs + + except NomaBlockedMessage as e: + guardrail_status = "guardrail_intervened" + guardrail_json_response = ( + response_json if isinstance(response_json, dict) else getattr(e, "detail", {"error": "blocked"}) + ) + raise + except Exception as e: + guardrail_status = "guardrail_failed_to_respond" + guardrail_json_response = str(e) + verbose_proxy_logger.error("Noma v2 guardrail failed: %s", str(e)) + if self.block_failures: + raise + return inputs + finally: + self._add_guardrail_observability( + request_data=request_data, + start_time=start_time, + guardrail_status=guardrail_status, + guardrail_json_response=guardrail_json_response, + ) diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py new file mode 100644 index 0000000000..3314c5ca2e --- /dev/null +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -0,0 +1,691 @@ +""" +Guardrails and policies usage endpoints for the dashboard. +GET /guardrails/usage/overview, /guardrails/usage/detail/:id, /guardrails/usage/logs +""" + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + + +# --- Response models --- + + +class UsageOverviewRow(BaseModel): + id: str + name: str + type: str + provider: str + requestsEvaluated: int + failRate: float + avgScore: Optional[float] + avgLatency: Optional[float] + status: str # healthy | warning | critical + trend: str # up | down | stable + + +class UsageOverviewResponse(BaseModel): + rows: List[UsageOverviewRow] + chart: List[Dict[str, Any]] # [{ date, passed, blocked }] + totalRequests: int + totalBlocked: int + passRate: float + + +class UsageDetailResponse(BaseModel): + guardrail_id: str + guardrail_name: str + type: str + provider: str + requestsEvaluated: int + failRate: float + avgScore: Optional[float] + avgLatency: Optional[float] + status: str + trend: str + description: Optional[str] + time_series: List[Dict[str, Any]] + + +class UsageLogEntry(BaseModel): + id: str + timestamp: str + action: str # blocked | passed | flagged + score: Optional[float] + latency_ms: Optional[float] + model: Optional[str] + input_snippet: Optional[str] + output_snippet: Optional[str] + reason: Optional[str] + + +class UsageLogsResponse(BaseModel): + logs: List[UsageLogEntry] + total: int + page: int + page_size: int + + +def _status_from_fail_rate(fail_rate: float) -> str: + if fail_rate > 15: + return "critical" + if fail_rate > 5: + return "warning" + return "healthy" + + +def _trend_from_comparison(current_fail: float, previous_fail: float) -> str: + if previous_fail <= 0: + return "stable" + diff = current_fail - previous_fail + if diff > 0.5: + return "up" + if diff < -0.5: + return "down" + return "stable" + + +def _aggregate_daily_metrics(metrics: Any, id_attr: str) -> Dict[str, Dict[str, Any]]: + agg: Dict[str, Dict[str, Any]] = {} + for m in metrics: + gid = getattr(m, id_attr) + if gid not in agg: + agg[gid] = {"requests": 0, "passed": 0, "blocked": 0, "flagged": 0} + agg[gid]["requests"] += int(m.requests_evaluated or 0) + agg[gid]["passed"] += int(m.passed_count or 0) + agg[gid]["blocked"] += int(m.blocked_count or 0) + agg[gid]["flagged"] += int(m.flagged_count or 0) + return agg + + +def _prev_fail_rates(metrics_prev: Any, id_attr: str) -> Dict[str, float]: + prev_agg_raw: Dict[str, Dict[str, int]] = {} + for m in metrics_prev: + gid = getattr(m, id_attr) + r, b = int(m.requests_evaluated or 0), int(m.blocked_count or 0) + if gid not in prev_agg_raw: + prev_agg_raw[gid] = {"req": 0, "blocked": 0} + prev_agg_raw[gid]["req"] += r + prev_agg_raw[gid]["blocked"] += b + return { + gid: (100.0 * v["blocked"] / v["req"]) if v["req"] else 0.0 + for gid, v in prev_agg_raw.items() + } + + +def _chart_from_metrics(metrics: Any) -> List[Dict[str, Any]]: + chart_by_date: Dict[str, Dict[str, int]] = {} + for m in metrics: + d = m.date + if d not in chart_by_date: + chart_by_date[d] = {"passed": 0, "blocked": 0} + chart_by_date[d]["passed"] += int(m.passed_count or 0) + chart_by_date[d]["blocked"] += int(m.blocked_count or 0) + return [ + {"date": d, "passed": v["passed"], "blocked": v["blocked"]} + for d, v in sorted(chart_by_date.items()) + ] + + +def _get_guardrail_attrs(g: Any) -> tuple[Any, str]: + """Get (guardrail_id, display_name) from guardrail - handles Prisma model or dict.""" + gid = getattr(g, "guardrail_id", None) or ( + g.get("guardrail_id") if isinstance(g, dict) else None + ) + name = getattr(g, "guardrail_name", None) or ( + g.get("guardrail_name") if isinstance(g, dict) else None + ) + return gid, (name or gid or "") + + +def _guardrail_overview_rows( + guardrails: Any, + agg: Dict[str, Dict[str, Any]], + prev_agg: Dict[str, float], +) -> List[UsageOverviewRow]: + rows: List[UsageOverviewRow] = [] + covered_keys: set = set() + for g in guardrails: + gid, display_name = _get_guardrail_attrs(g) + # Metrics are keyed by logical name from spend log metadata; guardrails table uses UUID + lookup_keys = [k for k in (display_name, gid) if k] + covered_keys.update(lookup_keys) + a = {"requests": 0, "passed": 0, "blocked": 0, "flagged": 0} + for k in lookup_keys: + if k in agg: + a = agg[k] + break + req, blocked = a["requests"], a["blocked"] + fail_rate = (100.0 * blocked / req) if req else 0.0 + litellm_params = ( + (g.litellm_params or {}) if isinstance(g.litellm_params, dict) else {} + ) + provider = str(litellm_params.get("guardrail", "Unknown")) + guardrail_info = ( + (g.guardrail_info or {}) if isinstance(g.guardrail_info, dict) else {} + ) + gtype = str(guardrail_info.get("type", "Guardrail")) + prev_fail = 0.0 + for k in lookup_keys: + if k in prev_agg: + prev_fail = float(prev_agg.get(k, 0.0) or 0.0) + break + trend = _trend_from_comparison(fail_rate, prev_fail) + rows.append( + UsageOverviewRow( + id=gid, + name=display_name or str(gid), + type=gtype, + provider=provider, + requestsEvaluated=req, + failRate=round(fail_rate, 1), + avgScore=None, + avgLatency=None, + status=_status_from_fail_rate(fail_rate), + trend=trend, + ) + ) + # Add rows for guardrails with metrics but not in guardrails table (e.g. MCP, config) + for agg_key, a in agg.items(): + if agg_key in covered_keys or a["requests"] == 0: + continue + req, blocked = a["requests"], a["blocked"] + fail_rate = (100.0 * blocked / req) if req else 0.0 + prev_fail = float(prev_agg.get(agg_key, 0.0) or 0.0) + trend = _trend_from_comparison(fail_rate, prev_fail) + rows.append( + UsageOverviewRow( + id=agg_key, + name=agg_key, + type="Guardrail", + provider="Custom", + requestsEvaluated=req, + failRate=round(fail_rate, 1), + avgScore=None, + avgLatency=None, + status=_status_from_fail_rate(fail_rate), + trend=trend, + ) + ) + return rows + + +def _policy_overview_rows( + policies: Any, + agg: Dict[str, Dict[str, Any]], + prev_agg: Dict[str, float], +) -> List[UsageOverviewRow]: + rows: List[UsageOverviewRow] = [] + for p in policies: + pid = p.policy_id + a = agg.get(pid, {"requests": 0, "passed": 0, "blocked": 0, "flagged": 0}) + req, blocked = a["requests"], a["blocked"] + fail_rate = (100.0 * blocked / req) if req else 0.0 + trend = _trend_from_comparison(fail_rate, prev_agg.get(pid, 0.0)) + rows.append( + UsageOverviewRow( + id=pid, + name=p.policy_name or pid, + type="Policy", + provider="LiteLLM", + requestsEvaluated=req, + failRate=round(fail_rate, 1), + avgScore=None, + avgLatency=None, + status=_status_from_fail_rate(fail_rate), + trend=trend, + ) + ) + return rows + + +@router.get( + "/guardrails/usage/overview", + tags=["Guardrails"], + dependencies=[Depends(user_api_key_auth)], + response_model=UsageOverviewResponse, +) +async def guardrails_usage_overview( + start_date: Optional[str] = Query(None, description="YYYY-MM-DD"), + end_date: Optional[str] = Query(None, description="YYYY-MM-DD"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Return guardrail performance overview for the dashboard.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return UsageOverviewResponse( + rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0 + ) + + now = datetime.now(timezone.utc) + end = end_date or now.strftime("%Y-%m-%d") + start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d") + + try: + # Guardrails from DB + guardrails = await prisma_client.db.litellm_guardrailstable.find_many() + + # Daily metrics in range + metrics = await prisma_client.db.litellm_dailyguardrailmetrics.find_many( + where={"date": {"gte": start, "lte": end}} + ) + + # Previous period for trend + start_prev = ( + datetime.strptime(start, "%Y-%m-%d") - timedelta(days=7) + ).strftime("%Y-%m-%d") + metrics_prev = await prisma_client.db.litellm_dailyguardrailmetrics.find_many( + where={"date": {"gte": start_prev, "lt": start}} + ) + + agg = _aggregate_daily_metrics(metrics, "guardrail_id") + prev_agg = _prev_fail_rates(metrics_prev, "guardrail_id") + chart = _chart_from_metrics(metrics) + total_requests = sum(a["requests"] for a in agg.values()) + total_blocked = sum(a["blocked"] for a in agg.values()) + pass_rate = ( + (100.0 * (total_requests - total_blocked) / total_requests) + if total_requests + else 100.0 + ) + rows = _guardrail_overview_rows(guardrails, agg, prev_agg) + return UsageOverviewResponse( + rows=rows, + chart=chart, + totalRequests=total_requests, + totalBlocked=total_blocked, + passRate=round(pass_rate, 1), + ) + except Exception as e: + from litellm.proxy.utils import handle_exception_on_proxy + + raise handle_exception_on_proxy(e) + + +@router.get( + "/guardrails/usage/detail/{guardrail_id}", + tags=["Guardrails"], + dependencies=[Depends(user_api_key_auth)], + response_model=UsageDetailResponse, +) +async def guardrails_usage_detail( + guardrail_id: str, + start_date: Optional[str] = Query(None), + end_date: Optional[str] = Query(None), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Return single guardrail usage metrics and time series.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + from fastapi import HTTPException + + raise HTTPException(status_code=500, detail="Prisma client not initialized") + + now = datetime.now(timezone.utc) + end = end_date or now.strftime("%Y-%m-%d") + start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d") + + guardrail = await prisma_client.db.litellm_guardrailstable.find_unique( + where={"guardrail_id": guardrail_id} + ) + if not guardrail: + from fastapi import HTTPException + + raise HTTPException(status_code=404, detail="Guardrail not found") + + # Metrics are keyed by logical name (from spend log metadata), not UUID + logical_id = getattr(guardrail, "guardrail_name", None) or ( + guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None + ) + metric_ids = [i for i in (logical_id, guardrail_id) if i] + + metrics = await prisma_client.db.litellm_dailyguardrailmetrics.find_many( + where={ + "guardrail_id": {"in": metric_ids}, + "date": {"gte": start, "lte": end}, + } + ) + metrics_prev = await prisma_client.db.litellm_dailyguardrailmetrics.find_many( + where={ + "guardrail_id": {"in": metric_ids}, + "date": {"lt": start}, + } + ) + + requests = sum(int(m.requests_evaluated or 0) for m in metrics) + blocked = sum(int(m.blocked_count or 0) for m in metrics) + fail_rate = (100.0 * blocked / requests) if requests else 0.0 + + prev_blocked = sum(int(m.blocked_count or 0) for m in metrics_prev) + prev_req = sum(int(m.requests_evaluated or 0) for m in metrics_prev) + prev_fail = (100.0 * prev_blocked / prev_req) if prev_req else 0.0 + trend = _trend_from_comparison(fail_rate, prev_fail) + + # Aggregate by date in case metrics exist under both UUID and logical name + ts_by_date: Dict[str, Dict[str, Any]] = {} + for m in metrics: + d = m.date + if d not in ts_by_date: + ts_by_date[d] = {"passed": 0, "blocked": 0} + ts_by_date[d]["passed"] += int(m.passed_count or 0) + ts_by_date[d]["blocked"] += int(m.blocked_count or 0) + time_series = [ + {"date": d, "passed": v["passed"], "blocked": v["blocked"], "score": None} + for d, v in sorted(ts_by_date.items()) + ] + _litellm_params = getattr(guardrail, "litellm_params", None) or ( + guardrail.get("litellm_params") if isinstance(guardrail, dict) else None + ) + litellm_params = ( + _litellm_params + if isinstance(_litellm_params, dict) + else {} + ) + _guardrail_info = getattr(guardrail, "guardrail_info", None) or ( + guardrail.get("guardrail_info") if isinstance(guardrail, dict) else None + ) + guardrail_info = ( + _guardrail_info + if isinstance(_guardrail_info, dict) + else {} + ) + _guardrail_name = getattr(guardrail, "guardrail_name", None) or ( + guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None + ) + + return UsageDetailResponse( + guardrail_id=guardrail_id, + guardrail_name=_guardrail_name or guardrail_id, + type=str(guardrail_info.get("type", "Guardrail")), + provider=str(litellm_params.get("guardrail", "Unknown")), + requestsEvaluated=requests, + failRate=round(fail_rate, 1), + avgScore=None, + avgLatency=None, + status=_status_from_fail_rate(fail_rate), + trend=trend, + description=guardrail_info.get("description"), + time_series=time_series, + ) + + +def _build_usage_logs_where( + guardrail_ids: Optional[List[str]], + policy_id: Optional[str], + start_date: Optional[str], + end_date: Optional[str], +) -> Dict[str, Any]: + where: Dict[str, Any] = {} + if guardrail_ids: + where["guardrail_id"] = ( + {"in": guardrail_ids} if len(guardrail_ids) > 1 else guardrail_ids[0] + ) + if policy_id: + where["policy_id"] = policy_id + if start_date or end_date: + st_filter: Dict[str, Any] = {} + if start_date: + sd = start_date.replace("Z", "+00:00").strip() + if "T" not in sd: + sd += "T00:00:00+00:00" + st_filter["gte"] = datetime.fromisoformat(sd) + if end_date: + ed = end_date.replace("Z", "+00:00").strip() + if "T" not in ed: + ed += "T23:59:59+00:00" + st_filter["lte"] = datetime.fromisoformat(ed) + where["start_time"] = st_filter + return where + + +def _usage_log_entry_from_row( + r: Any, sl: Any, action_filter: Optional[str] +) -> Optional[UsageLogEntry]: + meta = sl.metadata + if isinstance(meta, str): + try: + meta = json.loads(meta) + except Exception: + meta = {} + guardrail_info_list = (meta or {}).get("guardrail_information") or [] + entry_for_guardrail = None + for gi in guardrail_info_list: + if (gi.get("guardrail_id") or gi.get("guardrail_name")) == r.guardrail_id: + entry_for_guardrail = gi + break + action_val = "passed" + score_val = None + latency_val = None + reason_val = None + if entry_for_guardrail: + st = (entry_for_guardrail.get("guardrail_status") or "").lower() + if "intervened" in st or "block" in st: + action_val = "blocked" + elif "fail" in st or "error" in st: + action_val = "flagged" + duration = entry_for_guardrail.get("duration") + if duration is not None: + latency_val = round(float(duration) * 1000, 0) + score_val = entry_for_guardrail.get( + "confidence_score" + ) or entry_for_guardrail.get("risk_score") + if score_val is not None: + score_val = round(float(score_val), 2) + resp = entry_for_guardrail.get("guardrail_response") + if isinstance(resp, str): + reason_val = resp[:500] + elif isinstance(resp, dict): + reason_val = str(resp)[:500] + if action_filter and action_val != action_filter: + return None + ts = ( + sl.startTime.isoformat() + if hasattr(sl.startTime, "isoformat") + else str(sl.startTime) + ) + return UsageLogEntry( + id=r.request_id, + timestamp=ts, + action=action_val, + score=score_val, + latency_ms=latency_val, + model=sl.model, + input_snippet=_input_snippet_for_log(sl), + output_snippet=_snippet(sl.response), + reason=reason_val, + ) + + +def _snippet(text: Any, max_len: int = 200) -> Optional[str]: + if text is None: + return None + if isinstance(text, str): + s = text + elif isinstance(text, list): + parts = [] + for item in text: + if isinstance(item, dict) and "content" in item: + c = item["content"] + parts.append(c if isinstance(c, str) else str(c)) + else: + parts.append(str(item)) + s = " ".join(parts) + else: + s = str(text) + result = (s[:max_len] + "...") if len(s) > max_len else s + if result == "{}": + return None + return result + + +def _input_snippet_for_log(sl: Any) -> Optional[str]: + """Snippet for request input: prefer messages, fall back to proxy_server_request (same as drawer).""" + out = _snippet(sl.messages) + if out: + return out + psr = getattr(sl, "proxy_server_request", None) + if not psr: + return None + if isinstance(psr, str): + try: + psr = json.loads(psr) + except Exception: + return _snippet(psr) + if isinstance(psr, dict): + msgs = psr.get("messages") + if msgs is None and isinstance(psr.get("body"), dict): + msgs = psr["body"].get("messages") + out = _snippet(msgs) + if out: + return out + return _snippet(psr) + return _snippet(psr) + + +@router.get( + "/guardrails/usage/logs", + tags=["Guardrails"], + dependencies=[Depends(user_api_key_auth)], + response_model=UsageLogsResponse, +) +async def guardrails_usage_logs( + guardrail_id: Optional[str] = Query(None), + policy_id: Optional[str] = Query(None), + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=100), + action: Optional[str] = Query(None), + start_date: Optional[str] = Query(None), + end_date: Optional[str] = Query(None), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Return paginated run logs for a guardrail (or policy) from SpendLogs via index.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return UsageLogsResponse(logs=[], total=0, page=page, page_size=page_size) + + if not guardrail_id and not policy_id: + return UsageLogsResponse(logs=[], total=0, page=page, page_size=page_size) + + try: + # Index rows may store either guardrail_id (UUID) or guardrail_name from metadata. + # Query by both so we match regardless of which was written. + effective_guardrail_ids: List[str] = [guardrail_id] if guardrail_id else [] + if guardrail_id: + guardrail = await prisma_client.db.litellm_guardrailstable.find_unique( + where={"guardrail_id": guardrail_id} + ) + if guardrail: + logical_name = getattr(guardrail, "guardrail_name", None) + if logical_name and logical_name not in effective_guardrail_ids: + effective_guardrail_ids.append(logical_name) + + where = _build_usage_logs_where( + effective_guardrail_ids or None, policy_id, start_date, end_date + ) + index_rows = await prisma_client.db.litellm_spendlogguardrailindex.find_many( + where=where, + order={"start_time": "desc"}, + skip=(page - 1) * page_size, + take=page_size + 1, + ) + total = await prisma_client.db.litellm_spendlogguardrailindex.count(where=where) + request_ids = [r.request_id for r in index_rows[:page_size]] + if not request_ids: + return UsageLogsResponse( + logs=[], total=total, page=page, page_size=page_size + ) + spend_logs = await prisma_client.db.litellm_spendlogs.find_many( + where={"request_id": {"in": request_ids}} + ) + log_by_id = {s.request_id: s for s in spend_logs} + logs_out: List[UsageLogEntry] = [] + for r in index_rows[:page_size]: + sl = log_by_id.get(r.request_id) + if not sl: + continue + entry = _usage_log_entry_from_row(r, sl, action) + if entry is not None: + logs_out.append(entry) + return UsageLogsResponse( + logs=logs_out, total=total, page=page, page_size=page_size + ) + except Exception as e: + from litellm.proxy.utils import handle_exception_on_proxy + + raise handle_exception_on_proxy(e) + + +# --- Policy usage (same shape as guardrails; policy metrics populated when policy_run is in metadata) --- + + +@router.get( + "/policies/usage/overview", + tags=["Policies"], + dependencies=[Depends(user_api_key_auth)], + response_model=UsageOverviewResponse, +) +async def policies_usage_overview( + start_date: Optional[str] = Query(None, description="YYYY-MM-DD"), + end_date: Optional[str] = Query(None, description="YYYY-MM-DD"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Return policy performance overview for the dashboard.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return UsageOverviewResponse( + rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0 + ) + + now = datetime.now(timezone.utc) + end = end_date or now.strftime("%Y-%m-%d") + start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d") + + try: + policies = await prisma_client.db.litellm_policytable.find_many() + metrics = await prisma_client.db.litellm_dailypolicymetrics.find_many( + where={"date": {"gte": start, "lte": end}} + ) + metrics_prev = await prisma_client.db.litellm_dailypolicymetrics.find_many( + where={ + "date": { + "gte": ( + datetime.strptime(start, "%Y-%m-%d") - timedelta(days=7) + ).strftime("%Y-%m-%d"), + "lt": start, + } + } + ) + agg = _aggregate_daily_metrics(metrics, "policy_id") + prev_agg = _prev_fail_rates(metrics_prev, "policy_id") + chart = _chart_from_metrics(metrics) + total_requests = sum(a["requests"] for a in agg.values()) + total_blocked = sum(a["blocked"] for a in agg.values()) + pass_rate = ( + (100.0 * (total_requests - total_blocked) / total_requests) + if total_requests + else 100.0 + ) + rows = _policy_overview_rows(policies, agg, prev_agg) + return UsageOverviewResponse( + rows=rows, + chart=chart, + totalRequests=total_requests, + totalBlocked=total_blocked, + passRate=round(pass_rate, 1), + ) + except Exception as e: + from litellm.proxy.utils import handle_exception_on_proxy + + raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py new file mode 100644 index 0000000000..248f3a1987 --- /dev/null +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -0,0 +1,170 @@ +""" +Track guardrail and policy usage for the dashboard: upsert daily metrics and +insert into SpendLogGuardrailIndex when spend logs are written. +""" + +import json +from collections import defaultdict +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.utils import PrismaClient + + +def _guardrail_status_to_action(status: Optional[str]) -> str: + """Map StandardLogging guardrail_status to blocked/passed/flagged.""" + if not status: + return "passed" + s = (status or "").lower() + if "intervened" in s or "block" in s: + return "blocked" + if "fail" in s or "error" in s: + return "flagged" + return "passed" + + +def _parse_guardrail_info_from_payload(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + """Extract guardrail_information from spend log payload metadata.""" + meta = payload.get("metadata") + if not meta: + return [] + if isinstance(meta, str): + try: + meta = json.loads(meta) + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(meta, dict): + return [] + info = meta.get("guardrail_information") or meta.get( + "standard_logging_guardrail_information" + ) + if not isinstance(info, list): + return [] + return info + + +def _date_str(dt: datetime) -> str: + """YYYY-MM-DD in UTC.""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).strftime("%Y-%m-%d") + + +async def process_spend_logs_guardrail_usage( + prisma_client: PrismaClient, + logs_to_process: List[Dict[str, Any]], +) -> None: + """ + After spend logs are written: update DailyGuardrailMetrics and insert + SpendLogGuardrailIndex rows from guardrail_information in each payload. + """ + if not logs_to_process: + return + # Aggregate daily metrics by (guardrail_id, date). Latency/score metrics dropped. + daily_guardrail: Dict[tuple, Dict[str, Any]] = defaultdict( + lambda: { + "requests_evaluated": 0, + "passed_count": 0, + "blocked_count": 0, + "flagged_count": 0, + } + ) + index_rows: List[Dict[str, Any]] = [] + + for payload in logs_to_process: + request_id = payload.get("request_id") + start_time = payload.get("startTime") + if not request_id or not start_time: + continue + if isinstance(start_time, str): + try: + start_time = datetime.fromisoformat(start_time.replace("Z", "+00:00")) + except (ValueError, TypeError): + continue + date_key = _date_str(start_time) + + for entry in _parse_guardrail_info_from_payload(payload): + guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or "" + if not guardrail_id: + continue + key = (guardrail_id, date_key) + daily_guardrail[key]["requests_evaluated"] += 1 + action = _guardrail_status_to_action(entry.get("guardrail_status")) + if action == "passed": + daily_guardrail[key]["passed_count"] += 1 + elif action == "blocked": + daily_guardrail[key]["blocked_count"] += 1 + else: + daily_guardrail[key]["flagged_count"] += 1 + policy_id = entry.get("policy_id") + index_rows.append({ + "request_id": request_id, + "guardrail_id": guardrail_id, + "policy_id": policy_id, + "start_time": start_time, + }) + + if not daily_guardrail and not index_rows: + return + + try: + # Insert index rows (skip duplicates by request_id + guardrail_id) + if index_rows: + index_data = [] + for r in index_rows: + st = r["start_time"] + if isinstance(st, str): + try: + st = datetime.fromisoformat(st.replace("Z", "+00:00")) + except (ValueError, TypeError): + continue + index_data.append({ + "request_id": r["request_id"], + "guardrail_id": r["guardrail_id"], + "policy_id": r.get("policy_id"), + "start_time": st, + }) + try: + await prisma_client.db.litellm_spendlogguardrailindex.create_many( + data=index_data, + skip_duplicates=True, + ) + except Exception as e: + verbose_proxy_logger.debug( + "Guardrail usage tracking: index create_many skipped: %s", e + ) + + # Upsert daily guardrail metrics (counts only; latency/score dropped) + for (guardrail_id, date_key), agg in daily_guardrail.items(): + n = int(agg["requests_evaluated"]) + if n == 0: + continue + await prisma_client.db.litellm_dailyguardrailmetrics.upsert( + where={ + "guardrail_id_date": { + "guardrail_id": guardrail_id, + "date": date_key, + } + }, + data={ + "create": { + "guardrail_id": guardrail_id, + "date": date_key, + "requests_evaluated": n, + "passed_count": int(agg["passed_count"]), + "blocked_count": int(agg["blocked_count"]), + "flagged_count": int(agg["flagged_count"]), + }, + "update": { + "requests_evaluated": {"increment": n}, + "passed_count": {"increment": int(agg["passed_count"])}, + "blocked_count": {"increment": int(agg["blocked_count"])}, + "flagged_count": {"increment": int(agg["flagged_count"])}, + }, + }, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Guardrail usage tracking failed (non-fatal): %s", e + ) diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 0c820f6b78..000682bbf8 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -141,6 +141,50 @@ async def update_deployments_with_access_group( return models_updated +async def update_specific_deployments_with_access_group( + model_ids: List[str], + access_group: str, + prisma_client: PrismaClient, +) -> int: + """ + Update specific deployments (by model_id) to include the access group. + + Unlike update_deployments_with_access_group which tags ALL deployments sharing + a model_name, this function only tags the specific deployments identified by + their unique model_id. + """ + models_updated = 0 + for model_id in model_ids: + verbose_proxy_logger.debug( + f"Updating specific deployment model_id: {model_id}" + ) + deployment = await prisma_client.db.litellm_proxymodeltable.find_unique( + where={"model_id": model_id} + ) + if deployment is None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Deployment with model_id '{model_id}' not found in Database." + }, + ) + model_info = deployment.model_info or {} + updated_model_info, was_modified = add_access_group_to_deployment( + model_info=model_info, + access_group=access_group, + ) + if was_modified: + await prisma_client.db.litellm_proxymodeltable.update( + where={"model_id": model_id}, + data={"model_info": json.dumps(updated_model_info)}, + ) + models_updated += 1 + verbose_proxy_logger.debug( + f"Updated deployment {model_id} with access group: {access_group}" + ) + return models_updated + + def remove_access_group_from_deployment( model_info: Dict[str, Any], access_group: str ) -> Tuple[Dict[str, Any], bool]: @@ -263,24 +307,32 @@ async def create_model_group( detail={"error": "access_group is required and cannot be empty"}, ) - # Validation: Check if model_names list is provided and not empty - if not data.model_names or len(data.model_names) == 0: + # Validation: Check that at least one of model_names or model_ids is provided + has_model_names = data.model_names and len(data.model_names) > 0 + has_model_ids = data.model_ids and len(data.model_ids) > 0 + + if not has_model_names and not has_model_ids: raise HTTPException( status_code=400, - detail={"error": "model_names list is required and cannot be empty"}, + detail={"error": "Either model_names or model_ids must be provided and non-empty"}, ) - - # Validation: Check if all models exist in the router - all_valid, missing_models = validate_models_exist( - model_names=data.model_names, - llm_router=llm_router, - ) - - if not all_valid: - raise HTTPException( - status_code=400, - detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, + + # If model_ids is provided, use it (more precise targeting) + use_model_ids = has_model_ids + + # Validate model_names exist in router (only if using model_names path) + if not use_model_ids and has_model_names: + assert data.model_names is not None + all_valid, missing_models = validate_models_exist( + model_names=data.model_names, + llm_router=llm_router, ) + + if not all_valid: + raise HTTPException( + status_code=400, + detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, + ) # Check if database is connected if prisma_client is None: @@ -301,12 +353,21 @@ async def create_model_group( detail={"error": f"Access group '{data.access_group}' already exists. Use PUT /access_group/{data.access_group}/update to modify it."}, ) - # Update deployments using helper function - models_updated = await update_deployments_with_access_group( - model_names=data.model_names, - access_group=data.access_group, - prisma_client=prisma_client, - ) + # Update deployments using the appropriate method + if use_model_ids: + assert data.model_ids is not None + models_updated = await update_specific_deployments_with_access_group( + model_ids=data.model_ids, + access_group=data.access_group, + prisma_client=prisma_client, + ) + else: + assert data.model_names is not None + models_updated = await update_deployments_with_access_group( + model_names=data.model_names, + access_group=data.access_group, + prisma_client=prisma_client, + ) await clear_cache() @@ -317,6 +378,7 @@ async def create_model_group( return NewModelGroupResponse( access_group=data.access_group, model_names=data.model_names, + model_ids=data.model_ids, models_updated=models_updated, ) @@ -496,12 +558,17 @@ async def update_access_group( f"Updating access group: {access_group} with models: {data.model_names}" ) - # Validation: Check if model_names list is provided and not empty - if not data.model_names or len(data.model_names) == 0: + # Validation: Check that at least one of model_names or model_ids is provided + has_model_names = data.model_names and len(data.model_names) > 0 + has_model_ids = data.model_ids and len(data.model_ids) > 0 + + if not has_model_names and not has_model_ids: raise HTTPException( status_code=400, - detail={"error": "model_names list is required and cannot be empty"}, + detail={"error": "Either model_names or model_ids must be provided and non-empty"}, ) + + use_model_ids = has_model_ids # Validation: Check if access group exists try: @@ -521,17 +588,19 @@ async def update_access_group( detail={"error": f"Failed to check access group existence: {str(e)}"}, ) - # Validation: Check if all new models exist - all_valid, missing_models = validate_models_exist( - model_names=data.model_names, - llm_router=llm_router, - ) - - if not all_valid: - raise HTTPException( - status_code=400, - detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, + # Validation: Check if all new models exist (only if using model_names path) + if not use_model_ids and has_model_names: + assert data.model_names is not None + all_valid, missing_models = validate_models_exist( + model_names=data.model_names, + llm_router=llm_router, ) + + if not all_valid: + raise HTTPException( + status_code=400, + detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, + ) try: # Step 1: Remove access group from ALL DB deployments (skip config models) @@ -552,12 +621,21 @@ async def update_access_group( data={"model_info": json.dumps(updated_model_info)}, ) - # Step 2: Add access group to new model_names - models_updated = await update_deployments_with_access_group( - model_names=data.model_names, - access_group=access_group, - prisma_client=prisma_client, - ) + # Step 2: Add access group using the appropriate method + if use_model_ids: + assert data.model_ids is not None + models_updated = await update_specific_deployments_with_access_group( + model_ids=data.model_ids, + access_group=access_group, + prisma_client=prisma_client, + ) + else: + assert data.model_names is not None + models_updated = await update_deployments_with_access_group( + model_names=data.model_names, + access_group=access_group, + prisma_client=prisma_client, + ) # Clear cache and reload models to pick up the access group changes await clear_cache() @@ -569,6 +647,7 @@ async def update_access_group( return NewModelGroupResponse( access_group=access_group, model_names=data.model_names, + model_ids=data.model_ids, models_updated=models_updated, ) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 17e266bdea..5a1b31aebb 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -214,29 +214,41 @@ def process_sso_jwt_access_token( if isinstance(result, dict): result_team_ids: Optional[List[str]] = result.get("team_ids", []) if not result_team_ids: - team_ids = sso_jwt_handler.get_team_ids_from_jwt(access_token_payload) + team_ids = sso_jwt_handler.get_team_ids_from_jwt( + access_token_payload + ) result["team_ids"] = team_ids else: result_team_ids = getattr(result, "team_ids", []) if result else [] if not result_team_ids: - team_ids = sso_jwt_handler.get_team_ids_from_jwt(access_token_payload) + team_ids = sso_jwt_handler.get_team_ids_from_jwt( + access_token_payload + ) setattr(result, "team_ids", team_ids) # Extract user role from access token if not already set from UserInfo - existing_role = result.get("user_role") if isinstance(result, dict) else getattr(result, "user_role", None) + existing_role = ( + result.get("user_role") + if isinstance(result, dict) + else getattr(result, "user_role", None) + ) if existing_role is None: user_role: Optional[LitellmUserRoles] = None # Try role_mappings first (group-based role determination) if role_mappings is not None and role_mappings.roles: group_claim = role_mappings.group_claim - user_groups_raw: Any = get_nested_value(access_token_payload, group_claim) + user_groups_raw: Any = get_nested_value( + access_token_payload, group_claim + ) user_groups: List[str] = [] if isinstance(user_groups_raw, list): user_groups = [str(g) for g in user_groups_raw] elif isinstance(user_groups_raw, str): - user_groups = [g.strip() for g in user_groups_raw.split(",") if g.strip()] + user_groups = [ + g.strip() for g in user_groups_raw.split(",") if g.strip() + ] elif user_groups_raw is not None: user_groups = [str(user_groups_raw)] @@ -250,8 +262,12 @@ def process_sso_jwt_access_token( # Fallback: try GENERIC_USER_ROLE_ATTRIBUTE on the access token payload if user_role is None: - generic_user_role_attribute_name = os.getenv("GENERIC_USER_ROLE_ATTRIBUTE", "role") - user_role_from_token = get_nested_value(access_token_payload, generic_user_role_attribute_name) + generic_user_role_attribute_name = os.getenv( + "GENERIC_USER_ROLE_ATTRIBUTE", "role" + ) + user_role_from_token = get_nested_value( + access_token_payload, generic_user_role_attribute_name + ) if user_role_from_token is not None: user_role = get_litellm_user_role(user_role_from_token) verbose_proxy_logger.debug( @@ -309,7 +325,7 @@ async def google_login( total_users = await prisma_client.db.litellm_usertable.count() if total_users and total_users > 5: raise ProxyException( - message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat You are seeing this error message because You set one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, or `GENERIC_CLIENT_ID` in your env. Please unset this", + message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions You are seeing this error message because You set one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, or `GENERIC_CLIENT_ID` in your env. Please unset this", type=ProxyErrorTypes.auth_error, param="premium_user", code=status.HTTP_403_FORBIDDEN, @@ -662,13 +678,11 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: import ast try: - generic_user_role_mappings_data: Dict[ - LitellmUserRoles, List[str] - ] = ast.literal_eval(generic_role_mappings) + generic_user_role_mappings_data: Dict[LitellmUserRoles, List[str]] = ( + ast.literal_eval(generic_role_mappings) + ) if isinstance(generic_user_role_mappings_data, dict): - from litellm.types.proxy.management_endpoints.ui_sso import ( - RoleMappings, - ) + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings role_mappings_data = { "provider": "generic", @@ -770,7 +784,9 @@ async def get_generic_sso_response( ) access_token_str: Optional[str] = generic_sso.access_token - process_sso_jwt_access_token(access_token_str, sso_jwt_handler, result, role_mappings=role_mappings) + process_sso_jwt_access_token( + access_token_str, sso_jwt_handler, result, role_mappings=role_mappings + ) except Exception as e: verbose_proxy_logger.exception( @@ -1000,9 +1016,9 @@ def apply_user_info_values_to_sso_user_defined_values( else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: - user_defined_values[ - "user_role" - ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + user_defined_values["user_role"] = ( + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + ) verbose_proxy_logger.debug( "No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY" ) @@ -1430,9 +1446,9 @@ async def insert_sso_user( if user_defined_values.get("max_budget") is None: user_defined_values["max_budget"] = litellm.max_internal_user_budget if user_defined_values.get("budget_duration") is None: - user_defined_values[ - "budget_duration" - ] = litellm.internal_user_budget_duration + user_defined_values["budget_duration"] = ( + litellm.internal_user_budget_duration + ) if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -2533,9 +2549,9 @@ class MicrosoftSSOHandler: # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: - original_msft_result[ - MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY - ] = user_team_ids + original_msft_result[MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY] = ( + user_team_ids + ) original_msft_result["app_roles"] = app_roles return original_msft_result or {} @@ -2654,9 +2670,9 @@ class MicrosoftSSOHandler: # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[ - str - ] = MicrosoftSSOHandler.graph_api_user_groups_endpoint + next_link: Optional[str] = ( + MicrosoftSSOHandler.graph_api_user_groups_endpoint + ) auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 @@ -2885,7 +2901,7 @@ async def debug_sso_login(request: Request): ): if premium_user is not True: raise ProxyException( - message="You must be a LiteLLM Enterprise user to use SSO. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat You are seeing this error message because You set one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, or `GENERIC_CLIENT_ID` in your env. Please unset this", + message="You must be a LiteLLM Enterprise user to use SSO. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions You are seeing this error message because You set one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, or `GENERIC_CLIENT_ID` in your env. Please unset this", type=ProxyErrorTypes.auth_error, param="premium_user", code=status.HTTP_403_FORBIDDEN, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c1181dd52c..5d0c7a89d8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1,4 +1,3 @@ -import anyio import asyncio import copy import enum @@ -31,6 +30,7 @@ from typing import ( get_type_hints, ) +import anyio from pydantic import BaseModel, Json from litellm._uuid import uuid @@ -3161,6 +3161,7 @@ class ProxyConfig: alert_types=general_settings.get("alert_types", None), alert_to_webhook_url=general_settings.get("alert_to_webhook_url", None), alerting_args=general_settings.get("alerting_args", None), + alert_type_config=general_settings.get("alert_type_config", None), redis_cache=redis_usage_cache, ) @@ -3598,9 +3599,6 @@ class ProxyConfig: parsed = value elif isinstance(value, str): import json - - import yaml - try: parsed = yaml.safe_load(value) except (yaml.YAMLError, json.JSONDecodeError): @@ -4388,6 +4386,9 @@ class ProxyConfig: litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() + # Repopulate provider model sets (e.g. litellm.anthropic_models) so that + # wildcard patterns like "anthropic/*" include any newly added models. + litellm.add_known_models(model_cost_map=new_model_cost_map) # Update pod's in-memory last reload time last_model_cost_map_reload = current_time.isoformat() @@ -10769,6 +10770,81 @@ async def get_image(): return FileResponse(logo_path, media_type="image/jpeg") +@app.get("/get_favicon", include_in_schema=False) +async def get_favicon(): + """Get custom favicon for the admin UI.""" + from fastapi.responses import Response + + current_dir = os.path.dirname(os.path.abspath(__file__)) + default_favicon = os.path.join( + current_dir, "_experimental", "out", "favicon.ico" + ) + + favicon_url = os.getenv("LITELLM_FAVICON_URL", "") + + if not favicon_url: + if os.path.exists(default_favicon): + return FileResponse(default_favicon, media_type="image/x-icon") + raise HTTPException( + status_code=404, detail="Default favicon not found" + ) + + if favicon_url.startswith(("http://", "https://")): + try: + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + ) + from litellm.types.llms.custom_http import httpxSpecialProvider + + async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.UI, + params={"timeout": 5.0}, + ) + response = await async_client.get(favicon_url) + if response.status_code == 200: + content_type = response.headers.get( + "content-type", "image/x-icon" + ) + return Response( + content=response.content, + media_type=content_type, + ) + else: + verbose_proxy_logger.warning( + "Failed to fetch favicon from %s: status %s", + favicon_url, + response.status_code, + ) + if os.path.exists(default_favicon): + return FileResponse( + default_favicon, media_type="image/x-icon" + ) + raise HTTPException( + status_code=404, detail="Favicon not found" + ) + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.debug( + "Error downloading favicon from %s: %s", favicon_url, e + ) + if os.path.exists(default_favicon): + return FileResponse( + default_favicon, media_type="image/x-icon" + ) + raise HTTPException( + status_code=404, detail="Favicon not found" + ) + else: + if os.path.exists(favicon_url): + return FileResponse(favicon_url, media_type="image/x-icon") + if os.path.exists(default_favicon): + return FileResponse(default_favicon, media_type="image/x-icon") + raise HTTPException( + status_code=404, detail="Favicon not found" + ) + + #### INVITATION MANAGEMENT #### @@ -11890,6 +11966,9 @@ async def reload_model_cost_map( litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() + # Repopulate provider model sets (e.g. litellm.anthropic_models) so that + # wildcard patterns like "anthropic/*" include any newly added models. + litellm.add_known_models(model_cost_map=new_model_cost_map) # Update pod's in-memory last reload time global last_model_cost_map_reload @@ -12144,6 +12223,55 @@ async def get_model_cost_map_reload_status( ) +@router.get( + "/model/cost_map/source", + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) +async def get_model_cost_map_source( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + ADMIN ONLY / MASTER KEY Only Endpoint + + Returns information about where the current model cost/pricing data was loaded from. + + Response fields: + - source: "local" (bundled backup) or "remote" (fetched from URL) + - url: the remote URL that was attempted (null when env-forced local) + - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage + - fallback_reason: human-readable reason why remote failed (null on success) + - model_count: number of models in the currently loaded cost map + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail=f"Access denied. Admin role required. Current role: {user_api_key_dict.user_role}", + ) + + try: + from litellm.litellm_core_utils.get_model_cost_map import ( + get_model_cost_map_source_info, + ) + + source_info = get_model_cost_map_source_info() + model_count = len(litellm.model_cost) if litellm.model_cost else 0 + + return { + **source_info, + "model_count": model_count, + } + except Exception as e: + verbose_proxy_logger.exception( + f"Failed to get model cost map source info: {str(e)}" + ) + raise HTTPException( + status_code=500, + detail=f"Failed to get model cost map source info: {str(e)}", + ) + + #### ANTHROPIC BETA HEADERS RELOAD ENDPOINTS #### diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 6d60a218fd..29c9cb571c 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -2,8 +2,16 @@ import json import os from typing import List +import litellm from fastapi import APIRouter, Depends, HTTPException +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_blog_posts import ( + BlogPost, + BlogPostsResponse, + GetBlogPosts, + get_blog_posts, +) from litellm.proxy._types import CommonProxyErrors from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.agents import AgentCard @@ -193,6 +201,30 @@ async def get_litellm_model_cost_map(): ) +@router.get( + "/public/litellm_blog_posts", + tags=["public"], + response_model=BlogPostsResponse, +) +async def get_litellm_blog_posts(): + """ + Public endpoint to get the latest LiteLLM blog posts. + + Fetches from GitHub with a 1-hour in-process cache. + Falls back to the bundled local backup on any failure. + """ + try: + posts_data = get_blog_posts(url=litellm.blog_posts_url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: get_litellm_blog_posts endpoint fallback triggered: %s", str(e) + ) + posts_data = GetBlogPosts.load_local_blog_posts() + + posts = [BlogPost(**p) for p in posts_data[:5]] + return BlogPostsResponse(posts=posts) + + @router.get( "/public/agents/fields", tags=["public", "[beta] Agents"], diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 5d2cad6da5..50c0a55a87 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -273,7 +273,6 @@ model LiteLLM_MCPServerTable { alias String? description String? url String? - spec_path String? transport String @default("sse") auth_type String? credentials Json? @default("{}") @@ -813,6 +812,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t file_object Json // Stores the OpenAIFileObject file_purpose String // either 'batch' or 'fine-tune' status String? // check if batch cost has been tracked + batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed created_at DateTime @default(now()) created_by String? updated_at DateTime @updatedAt @@ -866,6 +866,54 @@ model LiteLLM_GuardrailsTable { updated_at DateTime @updatedAt } +// Daily guardrail metrics for usage dashboard (one row per guardrail per day) +model LiteLLM_DailyGuardrailMetrics { + guardrail_id String // logical id; may not FK if guardrail from config + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([guardrail_id, date]) + @@index([date]) + @@index([guardrail_id]) +} + +// Daily policy metrics for usage dashboard (one row per policy per day) +model LiteLLM_DailyPolicyMetrics { + policy_id String + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([policy_id, date]) + @@index([date]) + @@index([policy_id]) +} + +// Index for fast "last N logs for guardrail/policy" from SpendLogs +model LiteLLM_SpendLogGuardrailIndex { + request_id String + guardrail_id String + policy_id String? // set when run as part of a policy pipeline + start_time DateTime + + @@id([request_id, guardrail_id]) + @@index([guardrail_id, start_time]) + @@index([policy_id, start_time]) +} + // Prompt table for storing prompt configurations model LiteLLM_PromptTable { id String @id @default(uuid()) diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index b465d13bc7..74cff0315a 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -30,6 +30,12 @@ class UIThemeConfig(BaseModel): description="URL or path to custom logo image. Can be a local file path or HTTP/HTTPS URL", ) + # Favicon configuration + favicon_url: Optional[str] = Field( + default=None, + description="URL to custom favicon image. Must be an HTTP/HTTPS URL to a .ico, .png, or .svg file", + ) + class SettingsResponse(BaseModel): """Base response model for settings with values and schema information""" @@ -794,6 +800,27 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): del os.environ["UI_LOGO_PATH"] verbose_proxy_logger.debug("Removed UI_LOGO_PATH from environment") + # Update LITELLM_FAVICON_URL environment variable if favicon_url is provided + favicon_url = theme_data.get("favicon_url") + verbose_proxy_logger.debug(f"Updating favicon_url: {favicon_url}") + + if ( + favicon_url and isinstance(favicon_url, str) and favicon_url.strip() + ): # Check if favicon_url exists and is not empty/whitespace + config["environment_variables"]["LITELLM_FAVICON_URL"] = favicon_url + os.environ["LITELLM_FAVICON_URL"] = favicon_url + verbose_proxy_logger.debug(f"Set LITELLM_FAVICON_URL to: {favicon_url}") + else: + # Remove the environment variable to restore default favicon + if "LITELLM_FAVICON_URL" in config.get("environment_variables", {}): + del config["environment_variables"]["LITELLM_FAVICON_URL"] + verbose_proxy_logger.debug("Removed LITELLM_FAVICON_URL from config") + if "LITELLM_FAVICON_URL" in os.environ: + del os.environ["LITELLM_FAVICON_URL"] + verbose_proxy_logger.debug( + "Removed LITELLM_FAVICON_URL from environment" + ) + # Handle environment variable encryption if needed stored_config = config.copy() if ( @@ -809,7 +836,7 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): await proxy_config.save_config(new_config=stored_config) return { - "message": "Logo settings updated successfully.", + "message": "UI theme settings updated successfully.", "status": "success", "theme_config": theme_data, } diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f21fb0f9e3..c4ff325db1 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -23,31 +23,23 @@ from typing import ( ) from litellm import _custom_logger_compatible_callbacks_literal -from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME, MAX_TEAM_LIST_LIMIT -from litellm.proxy._types import ( - DB_CONNECTION_ERROR_TYPES, - CommonProxyErrors, - ProxyErrorTypes, - ProxyException, - SpendLogsMetadata, - SpendLogsPayload, -) +from litellm.constants import (DEFAULT_MODEL_CREATED_AT_TIME, + MAX_TEAM_LIST_LIMIT) +from litellm.proxy._types import (DB_CONNECTION_ERROR_TYPES, CommonProxyErrors, + ProxyErrorTypes, ProxyException, + SpendLogsMetadata, SpendLogsPayload) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypes, CallTypesLiteral try: - from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( - BaseEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( - ResendEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( - SendGridEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( - SMTPEmailLogger, - ) + from litellm_enterprise.enterprise_callbacks.send_emails.base_email import \ + BaseEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ + ResendEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ + SendGridEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ + SMTPEmailLogger except ImportError: BaseEmailLogger = None # type: ignore SendGridEmailLogger = None # type: ignore @@ -66,70 +58,56 @@ from fastapi import HTTPException, status import litellm import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging -from litellm import ( - EmbeddingResponse, - ImageResponse, - ModelResponse, - ModelResponseStream, - Router, -) +from litellm import (EmbeddingResponse, ImageResponse, ModelResponse, + ModelResponseStream, Router) from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import ( - CustomGuardrail, - ModifyResponseException, -) +from litellm.integrations.custom_guardrail import (CustomGuardrail, + ModifyResponseException) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting -from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.integrations.SlackAlerting.utils import \ + _add_langfuse_trace_id_to_alert from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import ( - AlertType, - CallInfo, - LiteLLM_VerificationTokenView, - Member, - UserAPIKeyAuth, -) +from litellm.proxy._types import (AlertType, CallInfo, + LiteLLM_VerificationTokenView, Member, + UserAPIKeyAuth) from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.db.create_views import ( - create_missing_views, - should_create_missing_views, -) +from litellm.proxy.db.create_views import (create_missing_views, + should_create_missing_views) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import PrismaWrapper -from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( - UnifiedLLMGuardrails, -) +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import \ + UnifiedLLMGuardrails from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter -from litellm.proxy.hooks.parallel_request_limiter import ( - _PROXY_MaxParallelRequestsHandler, -) +from litellm.proxy.hooks.parallel_request_limiter import \ + _PROXY_MaxParallelRequestsHandler from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES -from litellm.types.mcp import ( - MCPDuringCallResponseObject, - MCPPreCallRequestObject, - MCPPreCallResponseObject, -) -from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult +from litellm.types.mcp import (MCPDuringCallResponseObject, + MCPPreCallRequestObject, + MCPPreCallResponseObject) +from litellm.types.proxy.policy_engine.pipeline_types import \ + PipelineExecutionResult from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj Span = Union[_Span, Any] else: @@ -385,6 +363,7 @@ class ProxyLogging: alert_types: Optional[List[AlertType]] = None, alerting_args: Optional[dict] = None, alert_to_webhook_url: Optional[dict] = None, + alert_type_config: Optional[dict] = None, ): updated_slack_alerting: bool = False if alerting is not None: @@ -399,6 +378,8 @@ class ProxyLogging: if alert_to_webhook_url is not None: self.alert_to_webhook_url = alert_to_webhook_url updated_slack_alerting = True + if alert_type_config is not None: + updated_slack_alerting = True if updated_slack_alerting is True: self.slack_alerting_instance.update_values( @@ -407,6 +388,7 @@ class ProxyLogging: alert_types=self.alert_types, alerting_args=alerting_args, alert_to_webhook_url=self.alert_to_webhook_url, + alert_type_config=alert_type_config, ) if self.alerting is not None and "slack" in self.alerting: @@ -1068,10 +1050,9 @@ class ProxyLogging: """Process prompt template if applicable.""" from litellm.proxy.prompts.prompt_endpoints import ( - construct_versioned_prompt_id, - get_latest_version_prompt_id, - ) - from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + construct_versioned_prompt_id, get_latest_version_prompt_id) + from litellm.proxy.prompts.prompt_registry import \ + IN_MEMORY_PROMPT_REGISTRY from litellm.utils import get_non_default_completion_params if prompt_version is None: @@ -1121,9 +1102,8 @@ class ProxyLogging: def _process_guardrail_metadata(self, data: dict) -> None: """Process guardrails from metadata and add to applied_guardrails.""" - from litellm.proxy.common_utils.callback_utils import ( - add_guardrail_to_applied_guardrails_header, - ) + from litellm.proxy.common_utils.callback_utils import \ + add_guardrail_to_applied_guardrails_header metadata_standard = data.get("metadata") or {} metadata_litellm = data.get("litellm_metadata") or {} @@ -2020,7 +2000,8 @@ class ProxyLogging: if isinstance(response, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=response) elif isinstance(response, dict) and self.is_a2a_streaming_response(response): - from litellm.llms.a2a.common_utils import extract_text_from_a2a_response + from litellm.llms.a2a.common_utils import \ + extract_text_from_a2a_response response_str = extract_text_from_a2a_response(response) if response_str is not None: @@ -2029,7 +2010,8 @@ class ProxyLogging: _callback: Optional[CustomLogger] = None if isinstance(callback, CustomGuardrail): # Main - V2 Guardrails implementation - from litellm.types.guardrails import GuardrailEventHooks + from litellm.types.guardrails import \ + GuardrailEventHooks ## CHECK FOR MODEL-LEVEL GUARDRAILS modified_data = _check_and_merge_model_level_guardrails( @@ -4457,20 +4439,24 @@ class ProxyUpdateSpend: prisma_client: PrismaClient, db_writer_client: Optional[AsyncHTTPHandler], proxy_logging_obj: ProxyLogging, + logs_to_process: Optional[List[Dict[str, Any]]] = None, ): BATCH_SIZE = 1000 # Preferred size of each batch to write to the database MAX_LOGS_PER_INTERVAL = ( 10000 # Maximum number of logs to flush in a single interval ) - # Atomically read and remove logs to process (protected by lock) - async with prisma_client._spend_log_transactions_lock: - logs_to_process = prisma_client.spend_log_transactions[ - :MAX_LOGS_PER_INTERVAL - ] - # Remove the logs we're about to process - prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[ - len(logs_to_process) : - ] + popped_batch = False + if logs_to_process is None: + # Atomically read and remove logs to process (protected by lock) + async with prisma_client._spend_log_transactions_lock: + logs_to_process = prisma_client.spend_log_transactions[ + :MAX_LOGS_PER_INTERVAL + ] + # Remove the logs we're about to process + prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[ + len(logs_to_process) : + ] + popped_batch = True start_time = time.time() try: for i in range(n_retry_times + 1): @@ -4530,8 +4516,9 @@ class ProxyUpdateSpend: e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj ) finally: - # Clean up logs_to_process after all processing is complete - del logs_to_process + # Clean up logs_to_process only if we popped it (caller-owned otherwise) + if popped_batch: + del logs_to_process @staticmethod def disable_spend_updates() -> bool: @@ -4597,24 +4584,47 @@ async def update_spend_logs_job( Job to process spend_log_transactions queue. This job is triggered based on queue size rather than time. - Processes spend log transactions when the queue reaches a threshold. + Pops the batch once, writes spend logs, then runs guardrail usage tracking. """ n_retry_times = 3 + MAX_LOGS_PER_INTERVAL = 10000 - # Check queue size with lock protection + # Atomically pop batch from queue async with prisma_client._spend_log_transactions_lock: queue_size = len(prisma_client.spend_log_transactions) - if queue_size == 0: return + async with prisma_client._spend_log_transactions_lock: + logs_to_process = prisma_client.spend_log_transactions[ + :MAX_LOGS_PER_INTERVAL + ] + prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[ + len(logs_to_process) : + ] + await ProxyUpdateSpend.update_spend_logs( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, db_writer_client=db_writer_client, + logs_to_process=logs_to_process, ) + # Guardrail/policy usage tracking (same batch, outside spend-logs update) + try: + from litellm.proxy.guardrails.usage_tracking import \ + process_spend_logs_guardrail_usage + await process_spend_logs_guardrail_usage( + prisma_client=prisma_client, + logs_to_process=logs_to_process, + ) + except Exception as guardrail_tracking_err: + verbose_proxy_logger.debug( + "Guardrail usage tracking failed (non-fatal): %s", + guardrail_tracking_err, + ) + async def _monitor_spend_logs_queue( prisma_client: PrismaClient, @@ -4630,10 +4640,8 @@ async def _monitor_spend_logs_queue( db_writer_client: Optional HTTP handler for external spend logs endpoint proxy_logging_obj: Proxy logging object """ - from litellm.constants import ( - SPEND_LOG_QUEUE_POLL_INTERVAL, - SPEND_LOG_QUEUE_SIZE_THRESHOLD, - ) + from litellm.constants import (SPEND_LOG_QUEUE_POLL_INTERVAL, + SPEND_LOG_QUEUE_SIZE_THRESHOLD) threshold = SPEND_LOG_QUEUE_SIZE_THRESHOLD base_interval = SPEND_LOG_QUEUE_POLL_INTERVAL @@ -5154,12 +5162,11 @@ async def get_available_models_for_user( List of model names available to the user """ from litellm.proxy.auth.auth_checks import get_team_object - from litellm.proxy.auth.model_checks import ( - get_complete_model_list, - get_key_models, - get_team_models, - ) - from litellm.proxy.management_endpoints.team_endpoints import validate_membership + from litellm.proxy.auth.model_checks import (get_complete_model_list, + get_key_models, + get_team_models) + from litellm.proxy.management_endpoints.team_endpoints import \ + validate_membership # Get proxy model list and access groups if llm_router is None: diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index fea91ab6e4..c87d1076d0 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -58,6 +58,7 @@ class SupportedGuardrailIntegrations(Enum): MODEL_ARMOR = "model_armor" OPENAI_MODERATION = "openai_moderation" NOMA = "noma" + NOMA_V2 = "noma_v2" TOOL_PERMISSION = "tool_permission" ZSCALER_AI_GUARD = "zscaler_ai_guard" JAVELIN = "javelin" @@ -436,6 +437,10 @@ class PillarGuardrailConfigModel(BaseModel): class NomaGuardrailConfigModel(BaseModel): """Configuration parameters for the Noma Security guardrail""" + use_v2: Optional[bool] = Field( + default=False, + description="If True and guardrail='noma', route to the new Noma v2 implementation instead of the legacy implementation.", + ) application_id: Optional[str] = Field( default=None, description="Application ID for Noma Security. Defaults to 'litellm' if not provided", diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 856640638c..078e7953ad 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -1,13 +1,15 @@ import os from datetime import datetime as dt from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Set +from typing import Any, Dict, List, Literal, Optional, Set, Union from pydantic import BaseModel, Field from typing_extensions import TypedDict from litellm.types.utils import LiteLLMPydanticObjectBase +DEFAULT_DIGEST_INTERVAL = 86400 # 24 hours in seconds + SLACK_ALERTING_THRESHOLD_5_PERCENT = 0.05 SLACK_ALERTING_THRESHOLD_15_PERCENT = 0.15 MAX_OLDEST_HANGING_REQUESTS_TO_CHECK = 20 @@ -199,3 +201,30 @@ class HangingRequestData(BaseModel): key_alias: Optional[str] = None team_alias: Optional[str] = None alerting_metadata: Optional[dict] = None + + +class AlertTypeConfig(LiteLLMPydanticObjectBase): + """Per-alert-type configuration, including digest mode settings.""" + + digest: bool = Field( + default=False, + description="Enable digest mode for this alert type. When enabled, duplicate alerts are aggregated into a single summary message.", + ) + digest_interval: int = Field( + default=DEFAULT_DIGEST_INTERVAL, + description="Digest window in seconds. Alerts are aggregated within this interval. Default 24 hours.", + ) + + +class DigestEntry(TypedDict): + """Tracks an in-flight digest bucket for a unique (alert_type, model, api_base) combination.""" + + alert_type: str + request_model: str + api_base: str + first_message: str + level: str + count: int + start_time: dt + last_time: dt + webhook_url: Union[str, List[str]] diff --git a/litellm/types/interactions/generated.py b/litellm/types/interactions/generated.py index 72693e8f18..30e4ff4722 100644 --- a/litellm/types/interactions/generated.py +++ b/litellm/types/interactions/generated.py @@ -392,6 +392,7 @@ class Status3(Enum): COMPLETED = 'COMPLETED' FAILED = 'FAILED' CANCELLED = 'CANCELLED' + INCOMPLETE = 'INCOMPLETE' class ModelOption(RootModel[str]): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py index 2d6d4a4512..c6fd587abe 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py @@ -1,10 +1,15 @@ from typing import Optional -from pydantic import BaseModel, Field +from pydantic import Field from .base import GuardrailConfigModel + class NomaGuardrailConfigModel(GuardrailConfigModel): + use_v2: Optional[bool] = Field( + default=False, + description="If True and guardrail='noma', route to the new Noma v2 implementation.", + ) api_key: Optional[str] = Field( default=None, description="The Noma API key. Reads from NOMA_API_KEY env var if None.", @@ -21,3 +26,30 @@ class NomaGuardrailConfigModel(GuardrailConfigModel): @staticmethod def ui_friendly_name() -> str: return "Noma Security" + + +class NomaV2GuardrailConfigModel(GuardrailConfigModel): + api_key: Optional[str] = Field( + default=None, + description="The Noma API key. Reads from NOMA_API_KEY env var if None.", + ) + api_base: Optional[str] = Field( + default=None, + description="The Noma API base URL. Defaults to https://api.noma.security.", + ) + application_id: Optional[str] = Field( + default=None, + description="The Noma Application ID. Reads from NOMA_APPLICATION_ID env var if None.", + ) + monitor_mode: Optional[bool] = Field( + default=None, + description="When true, run guardrail checks in monitor mode.", + ) + block_failures: Optional[bool] = Field( + default=None, + description="When true, fail closed on Noma API errors.", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "Noma Security v2" diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index c488c46ecc..6f07e5c6de 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -21,17 +21,20 @@ class UpdateUsefulLinksRequest(BaseModel): class NewModelGroupRequest(BaseModel): access_group: str # The access group name (e.g., "production-models") - model_names: List[str] # Existing model groups to include (e.g., ["gpt-4", "claude-3"]) + model_names: Optional[List[str]] = None # Existing model groups to include - tags ALL deployments for each name + model_ids: Optional[List[str]] = None # Specific deployment IDs to tag (more precise than model_names) class NewModelGroupResponse(BaseModel): access_group: str - model_names: List[str] + model_names: Optional[List[str]] = None + model_ids: Optional[List[str]] = None models_updated: int # Number of models updated class UpdateModelGroupRequest(BaseModel): - model_names: List[str] # Updated list of model groups to include + model_names: Optional[List[str]] = None # Updated list of model groups to include - tags ALL deployments for each name + model_ids: Optional[List[str]] = None # Specific deployment IDs to tag (more precise than model_names) class DeleteModelGroupResponse(BaseModel): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7760a894c7..94c387dd22 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,47 +1,61 @@ import json import time from enum import Enum -from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, - Union) +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, Union from openai._models import BaseModel as OpenAIObject -from openai.types.audio.transcription_create_params import \ - FileTypes as FileTypes # type: ignore +from openai.types.audio.transcription_create_params import ( + FileTypes as FileTypes, # type: ignore +) from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion -from openai.types.completion_usage import (CompletionTokensDetails, - CompletionUsage, - PromptTokensDetails) +from openai.types.completion_usage import ( + CompletionTokensDetails, + CompletionUsage, + PromptTokensDetails, +) from openai.types.moderation import Categories as Categories -from openai.types.moderation import \ - CategoryAppliedInputTypes as CategoryAppliedInputTypes +from openai.types.moderation import ( + CategoryAppliedInputTypes as CategoryAppliedInputTypes, +) from openai.types.moderation import CategoryScores as CategoryScores from openai.types.moderation_create_response import Moderation as Moderation -from openai.types.moderation_create_response import \ - ModerationCreateResponse as ModerationCreateResponse +from openai.types.moderation_create_response import ( + ModerationCreateResponse as ModerationCreateResponse, +) from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from typing_extensions import Required, TypedDict from litellm._uuid import uuid -from litellm.types.llms.base import (BaseLiteLLMOpenAIResponseObject, - LiteLLMPydanticObjectBase) +from litellm.types.llms.base import ( + BaseLiteLLMOpenAIResponseObject, + LiteLLMPydanticObjectBase, +) from litellm.types.mcp import MCPServerCostInfo from ..litellm_core_utils.core_helpers import map_finish_reason from .agents import LiteLLMSendMessageResponse from .guardrails import GuardrailEventHooks -from .llms.anthropic_messages.anthropic_response import \ - AnthropicMessagesResponse +from .llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from .llms.base import HiddenParams -from .llms.openai import (AllMessageValues, Batch, ChatCompletionAnnotation, - ChatCompletionRedactedThinkingBlock, - ChatCompletionThinkingBlock, - ChatCompletionToolCallChunk, ChatCompletionToolParam, - ChatCompletionUsageBlock, FileSearchTool, - FineTuningJob, ImageURLListItem, - OpenAIChatCompletionChunk, - OpenAIChatCompletionFinishReason, OpenAIFileObject, - OpenAIRealtimeStreamList, ResponsesAPIResponse, - WebSearchOptions) +from .llms.openai import ( + AllMessageValues, + Batch, + ChatCompletionAnnotation, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, + ChatCompletionToolCallChunk, + ChatCompletionToolParam, + ChatCompletionUsageBlock, + FileSearchTool, + FineTuningJob, + ImageURLListItem, + OpenAIChatCompletionChunk, + OpenAIChatCompletionFinishReason, + OpenAIFileObject, + OpenAIRealtimeStreamList, + ResponsesAPIResponse, + WebSearchOptions, +) from .rerank import RerankResponse as RerankResponse if TYPE_CHECKING: @@ -212,6 +226,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): ] tpm: Optional[int] rpm: Optional[int] + provider_specific_entry: Optional[Dict[str, float]] class ModelInfo(ModelInfoBase, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 4771219d11..b4de1e00ef 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5385,14 +5385,18 @@ def _get_max_position_embeddings(model_name: str) -> Optional[int]: @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _cached_get_model_info_helper( - model: str, custom_llm_provider: Optional[str] + model: str, + custom_llm_provider: Optional[str], + api_base: Optional[str] = None, ) -> ModelInfoBase: """ _get_model_info_helper wrapped with lru_cache Speed Optimization to hit high RPS """ - return _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + return _get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) def get_provider_info( @@ -5428,7 +5432,9 @@ def _is_potential_model_name_in_model_cost( def _get_model_info_helper( # noqa: PLR0915 - model: str, custom_llm_provider: Optional[str] = None + model: str, + custom_llm_provider: Optional[str] = None, + api_base: Optional[str] = None, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -5486,7 +5492,7 @@ def _get_model_info_helper( # noqa: PLR0915 elif ( custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat" ) and not _is_potential_model_name_in_model_cost(potential_model_names): - return litellm.OllamaConfig().get_model_info(model) + return litellm.OllamaConfig().get_model_info(model, api_base=api_base) else: """ Check if: (in order of specificity) @@ -5719,11 +5725,12 @@ def _get_model_info_helper( # noqa: PLR0915 annotation_cost_per_page=_model_info.get( "annotation_cost_per_page", None ), + provider_specific_entry=_model_info.get( + "provider_specific_entry", None + ), ) except Exception as e: verbose_logger.debug(f"Error getting model info: {e}") - if "OllamaError" in str(e): - raise e raise Exception( "This model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json.".format( model, custom_llm_provider @@ -5732,7 +5739,11 @@ def _get_model_info_helper( # noqa: PLR0915 @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> ModelInfo: +def get_model_info( + model: str, + custom_llm_provider: Optional[str] = None, + api_base: Optional[str] = None, +) -> ModelInfo: """ Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token for a given model. @@ -5810,6 +5821,7 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod _model_info = _get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) provider_info = get_provider_info( @@ -7666,6 +7678,23 @@ def validate_and_fix_openai_tools(tools: Optional[List]) -> Optional[List[dict]] return new_tools +def validate_and_fix_thinking_param( + thinking: Optional["AnthropicThinkingParam"], +) -> Optional["AnthropicThinkingParam"]: + """ + Normalizes camelCase keys in the thinking param to snake_case. + Handles clients that send budgetTokens instead of budget_tokens. + """ + if thinking is None or not isinstance(thinking, dict): + return thinking + normalized = dict(thinking) + if "budgetTokens" in normalized and "budget_tokens" not in normalized: + normalized["budget_tokens"] = normalized.pop("budgetTokens") + elif "budgetTokens" in normalized and "budget_tokens" in normalized: + normalized.pop("budgetTokens") + return cast("AnthropicThinkingParam", normalized) + + def cleanup_none_field_in_message(message: AllMessageValues): """ Cleans up the message by removing the none field. diff --git a/litellm/videos/main.py b/litellm/videos/main.py index db09ab04f1..2225b9eec7 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -273,6 +273,7 @@ def video_content( video_id: str, timeout: Optional[float] = None, custom_llm_provider: Optional[str] = None, + variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Optional[Dict[str, Any]] = None, @@ -367,6 +368,7 @@ def video_content( extra_headers=extra_headers, client=kwargs.get("client"), _is_async=_is_async, + variant=variant, ) except Exception as e: @@ -385,6 +387,7 @@ async def avideo_content( video_id: str, timeout: Optional[float] = None, custom_llm_provider: Optional[str] = None, + variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Optional[Dict[str, Any]] = None, @@ -422,6 +425,7 @@ async def avideo_content( video_id=video_id, timeout=timeout, custom_llm_provider=custom_llm_provider, + variant=variant, extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a8de7f2056..39f3c7ce1c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8295,37 +8295,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "us/claude-sonnet-4-6": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, - "cache_read_input_token_cost": 3.3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, - "input_cost_per_token": 3.3e-06, - "input_cost_per_token_above_200k_tokens": 6.6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_200k_tokens": 2.475e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "inference_geo": "us" - }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -8517,100 +8486,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "us/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/us/claude-opus-4-6": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -8641,69 +8521,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "fast/claude-opus-4-6-20260205": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 3e-05, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "us/claude-opus-4-6-20260205": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -14768,7 +14590,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14819,7 +14648,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -14919,7 +14755,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -14963,7 +14806,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -15014,7 +14862,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "vertex_ai/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -15065,7 +14920,14 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 1.25e-07, @@ -16860,6 +16722,8 @@ "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_priority": 1.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 2.5e-06, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -16873,8 +16737,11 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, + "output_cost_per_token_priority": 1e-05, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-05, "rpm": 2000, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_service_tier": true, "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -16979,7 +16846,14 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -17027,7 +16901,12 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -17078,7 +16957,14 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -17129,7 +17015,14 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, - "tpm": 800000 + "tpm": 800000, + "input_cost_per_token_priority": 3.6e-06, + "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, + "output_cost_per_token_priority": 2.16e-05, + "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, + "cache_read_input_token_cost_priority": 3.6e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "supports_service_tier": true }, "gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -17175,7 +17068,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "input_cost_per_token_priority": 9e-07, + "input_cost_per_audio_token_priority": 1.8e-06, + "output_cost_per_token_priority": 5.4e-06, + "cache_read_input_token_cost_priority": 9e-08, + "supports_service_tier": true }, "gemini/gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 0.0, @@ -37808,4 +37706,4 @@ "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." } } -} +} \ No newline at end of file diff --git a/policy_templates.json b/policy_templates.json index 7c409d5c4f..6125650cb3 100644 --- a/policy_templates.json +++ b/policy_templates.json @@ -2013,5 +2013,367 @@ "Brand Protection" ], "estimated_latency_ms": 1 + }, + { + "id": "pdpa-singapore", + "title": "Singapore PDPA \u2014 Personal Data Protection", + "description": "Singapore Personal Data Protection Act (PDPA) compliance. Covers 5 obligation areas: personal identifier collection (s.13 Consent), sensitive data profiling (Advisory Guidelines), Do Not Call Registry violations (Part IX), overseas data transfers (s.26), and automated profiling without human oversight (Model AI Governance Framework). Also includes regex-based PII detection for NRIC/FIN, Singapore phone numbers, postal codes, passports, UEN, and bank account numbers. Zero-cost keyword-based detection.", + "icon": "ShieldCheckIcon", + "iconColor": "text-red-500", + "iconBg": "bg-red-50", + "guardrails": [ + "pdpa-sg-pii-identifiers", + "pdpa-sg-contact-information", + "pdpa-sg-financial-data", + "pdpa-sg-business-identifiers", + "pdpa-sg-personal-identifiers", + "pdpa-sg-sensitive-data", + "pdpa-sg-do-not-call", + "pdpa-sg-data-transfer", + "pdpa-sg-profiling-automated-decisions" + ], + "complexity": "High", + "guardrailDefinitions": [ + { + "guardrail_name": "pdpa-sg-pii-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_nric", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "passport_singapore", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore NRIC/FIN and passport numbers for PDPA compliance" + } + }, + { + "guardrail_name": "pdpa-sg-contact-information", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_phone", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "sg_postal_code", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "email", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore phone numbers, postal codes, and email addresses" + } + }, + { + "guardrail_name": "pdpa-sg-financial-data", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_bank_account", + "action": "MASK" + }, + { + "pattern_type": "prebuilt", + "pattern_name": "credit_card", + "action": "MASK" + } + ], + "pattern_redaction_format": "[{pattern_name}_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore bank account numbers and credit card numbers" + } + }, + { + "guardrail_name": "pdpa-sg-business-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "patterns": [ + { + "pattern_type": "prebuilt", + "pattern_name": "sg_uen", + "action": "MASK" + } + ], + "pattern_redaction_format": "[UEN_REDACTED]" + }, + "guardrail_info": { + "description": "Masks Singapore Unique Entity Numbers (business registration)" + } + }, + { + "guardrail_name": "pdpa-sg-personal-identifiers", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_personal_identifiers", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_personal_identifiers.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA s.13 \u2014 Blocks unauthorized collection, harvesting, or extraction of Singapore personal identifiers (NRIC/FIN, SingPass, passports)" + } + }, + { + "guardrail_name": "pdpa-sg-sensitive-data", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_sensitive_data", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_sensitive_data.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA Advisory Guidelines \u2014 Blocks profiling or inference of sensitive personal data categories (race, religion, health, politics) for Singapore residents" + } + }, + { + "guardrail_name": "pdpa-sg-do-not-call", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_do_not_call", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_do_not_call.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA Part IX \u2014 Blocks generation of unsolicited marketing lists and DNC Registry bypass attempts for Singapore phone numbers" + } + }, + { + "guardrail_name": "pdpa-sg-data-transfer", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_data_transfer", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_data_transfer.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA s.26 \u2014 Blocks unprotected overseas transfer of Singapore personal data without adequate safeguards" + } + }, + { + "guardrail_name": "pdpa-sg-profiling-automated-decisions", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_pdpa_profiling_automated_decisions", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_pdpa_profiling_automated_decisions.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "PDPA + Model AI Governance Framework \u2014 Blocks automated profiling and decision-making about Singapore residents without human oversight" + } + } + ], + "templateData": { + "policy_name": "pdpa-singapore", + "description": "Singapore PDPA compliance policy. Covers personal identifier protection (s.13), sensitive data profiling (Advisory Guidelines), Do Not Call Registry (Part IX), overseas data transfers (s.26), and automated profiling (Model AI Governance Framework). Includes regex-based PII detection for NRIC/FIN, phone numbers, postal codes, passports, UEN, and bank accounts.", + "guardrails_add": [ + "pdpa-sg-pii-identifiers", + "pdpa-sg-contact-information", + "pdpa-sg-financial-data", + "pdpa-sg-business-identifiers", + "pdpa-sg-personal-identifiers", + "pdpa-sg-sensitive-data", + "pdpa-sg-do-not-call", + "pdpa-sg-data-transfer", + "pdpa-sg-profiling-automated-decisions" + ], + "guardrails_remove": [] + }, + "tags": [ + "PII Protection", + "Regulatory", + "Singapore" + ], + "estimated_latency_ms": 1 + }, + { + "id": "mas-ai-risk-management", + "title": "Singapore MAS \u2014 AI Risk Management for Financial Institutions", + "description": "Monetary Authority of Singapore (MAS) AI Risk Management for Financial Institutions alignment. Covers 5 enforceable obligation areas: fairness & bias in financial decisions, transparency & explainability of AI models, human oversight for consequential actions, data governance for financial customer data, and model security against adversarial attacks. Based on Guidelines on Artificial Intelligence Risk Management (MAS), and aligned with the 2018 FEAT Principles and Project MindForge. Zero-cost keyword-based detection.", + "icon": "ShieldCheckIcon", + "iconColor": "text-blue-600", + "iconBg": "bg-blue-50", + "guardrails": [ + "mas-sg-fairness-bias", + "mas-sg-transparency-explainability", + "mas-sg-human-oversight", + "mas-sg-data-governance", + "mas-sg-model-security" + ], + "complexity": "High", + "guardrailDefinitions": [ + { + "guardrail_name": "mas-sg-fairness-bias", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_fairness_bias", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_fairness_bias.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks discriminatory AI practices in financial services that score, deny, or price based on protected attributes (race, religion, age, gender, nationality)" + } + }, + { + "guardrail_name": "mas-sg-transparency-explainability", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_transparency_explainability", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks deployment of opaque or unexplainable AI systems for consequential financial decisions" + } + }, + { + "guardrail_name": "mas-sg-human-oversight", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_human_oversight", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_human_oversight.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks fully automated financial AI decisions without human-in-the-loop for consequential actions (loans, claims, trading)" + } + }, + { + "guardrail_name": "mas-sg-data-governance", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_data_governance", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_data_governance.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks unauthorized sharing, exposure, or mishandling of financial customer data without proper governance and data lineage" + } + }, + { + "guardrail_name": "mas-sg-model-security", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "categories": [ + { + "category": "sg_mas_model_security", + "category_file": "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_model_security.yaml", + "enabled": true, + "action": "BLOCK", + "severity_threshold": "medium" + } + ] + }, + "guardrail_info": { + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) β€” Blocks adversarial attacks, model poisoning, inversion, and exfiltration attempts targeting financial AI systems" + } + } + ], + "templateData": { + "policy_name": "mas-ai-risk-management", + "description": "Guidelines on Artificial Intelligence Risk Management (MAS) for Financial Institutions alignment. Covers fairness & bias, transparency & explainability, human oversight, data governance, and model security. Aligned with the 2018 FEAT Principles, Project MindForge, and NIST AI RMF.", + "guardrails_add": [ + "mas-sg-fairness-bias", + "mas-sg-transparency-explainability", + "mas-sg-human-oversight", + "mas-sg-data-governance", + "mas-sg-model-security" + ], + "guardrails_remove": [] + }, + "tags": [ + "Financial Services", + "Regulatory", + "Singapore" + ], + "estimated_latency_ms": 1 } ] diff --git a/ruff.toml b/ruff.toml index a310446671..43ff802a68 100644 --- a/ruff.toml +++ b/ruff.toml @@ -12,4 +12,7 @@ exclude = ["litellm/types/*", "litellm/__init__.py", "litellm/proxy/example_conf "litellm/llms/anthropic/chat/__init__.py" = ["F401"] "litellm/llms/azure_ai/embed/__init__.py" = ["F401"] "litellm/llms/azure_ai/rerank/__init__.py" = ["F401"] -"litellm/llms/bedrock/chat/__init__.py" = ["F401"] \ No newline at end of file +"litellm/llms/bedrock/chat/__init__.py" = ["F401"] +"litellm/proxy/utils.py" = ["F401", "PLR0915"] +"litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py" = ["PLR0915"] +"litellm/proxy/guardrails/guardrail_hooks/guardrail_benchmarks/test_eval.py" = ["PLR0915"] diff --git a/schema.prisma b/schema.prisma index 5d2cad6da5..4af7484148 100644 --- a/schema.prisma +++ b/schema.prisma @@ -813,6 +813,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t file_object Json // Stores the OpenAIFileObject file_purpose String // either 'batch' or 'fine-tune' status String? // check if batch cost has been tracked + batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed created_at DateTime @default(now()) created_by String? updated_at DateTime @updatedAt @@ -866,6 +867,54 @@ model LiteLLM_GuardrailsTable { updated_at DateTime @updatedAt } +// Daily guardrail metrics for usage dashboard (one row per guardrail per day) +model LiteLLM_DailyGuardrailMetrics { + guardrail_id String // logical id; may not FK if guardrail from config + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([guardrail_id, date]) + @@index([date]) + @@index([guardrail_id]) +} + +// Daily policy metrics for usage dashboard (one row per policy per day) +model LiteLLM_DailyPolicyMetrics { + policy_id String + date String // YYYY-MM-DD + requests_evaluated BigInt @default(0) + passed_count BigInt @default(0) + blocked_count BigInt @default(0) + flagged_count BigInt @default(0) + avg_score Float? + avg_latency_ms Float? + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([policy_id, date]) + @@index([date]) + @@index([policy_id]) +} + +// Index for fast "last N logs for guardrail/policy" from SpendLogs +model LiteLLM_SpendLogGuardrailIndex { + request_id String + guardrail_id String + policy_id String? // set when run as part of a policy pipeline + start_time DateTime + + @@id([request_id, guardrail_id]) + @@index([guardrail_id, start_time]) + @@index([policy_id, start_time]) +} + // Prompt table for storing prompt configurations model LiteLLM_PromptTable { id String @id @default(uuid()) diff --git a/scripts/benchmark_mock.py b/scripts/benchmark_mock.py new file mode 100644 index 0000000000..057002883f --- /dev/null +++ b/scripts/benchmark_mock.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Quick benchmark for network_mock proxy overhead measurement.""" + +import argparse +import asyncio +import time +import statistics + +import aiohttp + + +REQUEST_BODY = { + "model": "db-openai-endpoint", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 100, + "user": "new_user", +} + +HEADERS = { + "Authorization": "Bearer sk-1234", + "Content-Type": "application/json", +} + + +async def send_request(session, url, semaphore): + async with semaphore: + start = time.perf_counter() + try: + async with session.post(url, json=REQUEST_BODY, headers=HEADERS) as resp: + await resp.read() + elapsed = time.perf_counter() - start + return elapsed if resp.status == 200 else None + except Exception: + return None + + +async def run_benchmark(url, n_requests, max_concurrent): + semaphore = asyncio.Semaphore(max_concurrent) + connector_limit = min(max_concurrent * 2, 200) + connector = aiohttp.TCPConnector( + limit=connector_limit, + limit_per_host=max_concurrent, + force_close=False, + enable_cleanup_closed=True, + ) + async with aiohttp.ClientSession(connector=connector) as session: + # warmup + await asyncio.gather(*[send_request(session, url, semaphore) for _ in range(min(50, n_requests))]) + + # timed run + wall_start = time.perf_counter() + results = await asyncio.gather(*[send_request(session, url, semaphore) for _ in range(n_requests)]) + wall_elapsed = time.perf_counter() - wall_start + + latencies = [r for r in results if r is not None] + failures = sum(1 for r in results if r is None) + + if not latencies: + return { + "mean": 0, "p50": 0, "p95": 0, "p99": 0, + "throughput": 0, "failures": n_requests, + "wall_time": wall_elapsed, "n_requests": n_requests, + "max_concurrent": max_concurrent, "latencies": [], + } + + latencies.sort() + n = len(latencies) + mean = statistics.mean(latencies) * 1000 + p50 = latencies[n // 2] * 1000 + p95 = latencies[int(n * 0.95)] * 1000 + p99 = latencies[int(n * 0.99)] * 1000 + throughput = n_requests / wall_elapsed + + return { + "mean": mean, "p50": p50, "p95": p95, "p99": p99, + "throughput": throughput, "failures": failures, + "wall_time": wall_elapsed, "n_requests": n_requests, + "max_concurrent": max_concurrent, "latencies": latencies, + } + + +def print_run_results(run_num, total_runs, result): + label = f" Run {run_num}/{total_runs}" if total_runs > 1 else " Results" + print(f"\n{'='*60}") + print(label) + print(f"{'='*60}") + print(f" Requests: {result['n_requests']} (failures: {result['failures']})") + print(f" Concurrency: {result['max_concurrent']}") + print(f" Wall time: {result['wall_time']:.2f}s") + print(f" Throughput: {result['throughput']:.0f} req/s") + print(f" Mean: {result['mean']:.2f} ms") + print(f" P50: {result['p50']:.2f} ms") + print(f" P95: {result['p95']:.2f} ms") + print(f" P99: {result['p99']:.2f} ms") + + +def print_aggregate(results): + all_latencies = [] + for r in results: + all_latencies.extend(r["latencies"]) + all_latencies.sort() + + total_failures = sum(r["failures"] for r in results) + total_requests = sum(r["n_requests"] for r in results) + n = len(all_latencies) + + if not all_latencies: + print(f"\n Aggregate: all {total_requests} requests failed across {len(results)} runs") + return + + mean = statistics.mean(all_latencies) * 1000 + p50 = all_latencies[n // 2] * 1000 + p95 = all_latencies[int(n * 0.95)] * 1000 + p99 = all_latencies[int(n * 0.99)] * 1000 + avg_throughput = statistics.mean(r["throughput"] for r in results) + + print(f"\n{'='*60}") + print(f" Aggregate ({len(results)} runs, {total_requests} total requests)") + print(f"{'='*60}") + print(f" Failures: {total_failures}") + print(f" Throughput: {avg_throughput:.0f} req/s (avg across runs)") + print(f" Mean: {mean:.2f} ms") + print(f" P50: {p50:.2f} ms") + print(f" P95: {p95:.2f} ms") + print(f" P99: {p99:.2f} ms") + + # Run-to-run variance + run_means = [r["mean"] for r in results] + run_throughputs = [r["throughput"] for r in results] + if len(run_means) > 1: + cov_latency = statistics.stdev(run_means) / statistics.mean(run_means) * 100 + cov_throughput = statistics.stdev(run_throughputs) / statistics.mean(run_throughputs) * 100 + print(f"\n Run-to-run variance:") + print(f" Latency CoV: {cov_latency:.1f}%") + print(f" Throughput CoV: {cov_throughput:.1f}%") + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="http://localhost:4000/chat/completions") + parser.add_argument("--requests", type=int, default=2000) + parser.add_argument("--max-concurrent", type=int, default=200) + parser.add_argument("--runs", type=int, default=1) + args = parser.parse_args() + + print(f"Benchmarking {args.url}") + print(f" {args.requests} requests, {args.max_concurrent} concurrency, {args.runs} run(s)") + + results = [] + for run_num in range(1, args.runs + 1): + result = await run_benchmark(args.url, args.requests, args.max_concurrent) + results.append(result) + print_run_results(run_num, args.runs, result) + + if args.runs > 1: + print_aggregate(results) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py new file mode 100644 index 0000000000..8a5be36354 --- /dev/null +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -0,0 +1,416 @@ +""" +Test Guidelines on Artificial Intelligence Risk Management (MAS) β€” Conditional Keyword Matching + +Tests 5 sub-guardrails covering Guidelines on Artificial Intelligence Risk Management (MAS) obligations +for Singapore financial institutions: + 1. sg_mas_fairness_bias β€” Discriminatory financial AI + 2. sg_mas_transparency_explainability β€” Opaque/unexplainable AI decisions + 3. sg_mas_human_oversight β€” Automated decisions without human review + 4. sg_mas_data_governance β€” Financial data mishandling + 5. sg_mas_model_security β€” Adversarial attacks on financial AI +""" +import sys +import os +import pytest + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, +) +from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter import ( + ContentFilterCategoryConfig, +) + + +# ── helpers ────────────────────────────────────────────────────────────── + +POLICY_DIR = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../litellm/proxy/guardrails/guardrail_hooks/" + "litellm_content_filter/policy_templates", + ) +) + + +def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuardrail: + path = os.path.join(POLICY_DIR, yaml_filename) + categories = [ + ContentFilterCategoryConfig( + category=category_name, + category_file=path, + enabled=True, + action="BLOCK", + severity_threshold="medium", + ) + ] + return ContentFilterGuardrail( + guardrail_name=f"mas-{category_name}-test", + categories=categories, + event_hook=litellm.types.guardrails.GuardrailEventHooks.pre_call, + ) + + +async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): + request_data = {"messages": [{"role": "user", "content": sentence}]} + with pytest.raises(Exception) as exc_info: + await guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + err = str(exc_info.value).lower() + assert "blocked" in err or "violation" in err, ( + f"Expected BLOCK for '{sentence}' ({reason}) but got: {exc_info.value}" + ) + + +async def _expect_allow(guardrail: ContentFilterGuardrail, sentence: str, reason: str): + request_data = {"messages": [{"role": "user", "content": sentence}]} + result = await guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + assert result is None or result["texts"][0] == sentence, ( + f"Expected ALLOW for '{sentence}' ({reason}) but it was blocked/modified" + ) + + +# ═══════════════════════════════════════════════════════════════════════ +# 1. Fairness & Bias +# ═══════════════════════════════════════════════════════════════════════ + +FAIRNESS_BIAS_CASES = [ + # ALWAYS BLOCK + ("deny loan based on race", "BLOCK", "always: deny loan based on race"), + ("credit score by ethnicity", "BLOCK", "always: credit score by ethnicity"), + ("insurance premium based on religion", "BLOCK", "always: insurance premium based on religion"), + ("reject application based on nationality", "BLOCK", "always: reject based on nationality"), + ("higher interest rate for Malay", "BLOCK", "always: higher rate for Malay"), + ("lower credit limit for Indian", "BLOCK", "always: lower credit limit for Indian"), + ("penalize based on gender", "BLOCK", "always: penalize based on gender"), + ("discriminate by age in lending", "BLOCK", "always: discriminate by age"), + ("price insurance by race Singapore", "BLOCK", "always: price insurance by race"), + ("exclude foreign worker from financial service", "BLOCK", "always: exclude foreign worker"), + + # CONDITIONAL: identifier + block word + ("deny credit approval based on ethnicity", "BLOCK", "conditional: deny + ethnicity"), + ("score by race for loan pricing", "BLOCK", "conditional: score by + race"), + ("segment by religion for insurance premium", "BLOCK", "conditional: segment by + religion"), + ("adjust by gender the interest rate", "BLOCK", "conditional: adjust by + gender"), + ("restrict access to loan approval based on age", "BLOCK", "conditional: restrict access + age"), + + # EXCEPTIONS + ("fairness audit of credit scoring model", "ALLOW", "exception: fairness audit"), + ("bias detection in lending algorithm", "ALLOW", "exception: bias detection"), + ("explain what FEAT principles are", "ALLOW", "exception: explain + FEAT principles"), + ("anti-discrimination policy for insurance", "ALLOW", "exception: anti-discrimination"), + ("research on racial bias in credit scoring", "ALLOW", "exception: research"), + + # NO MATCH β€” identifier only + ("race results for the marathon", "ALLOW", "no match: identifier only"), + ("age of the company", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("deny the request due to timeout", "ALLOW", "no match: block word only"), + ("credit score improvement tips", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def fairness_guardrail(): + return _make_guardrail("sg_mas_fairness_bias.yaml", "sg_mas_fairness_bias") + + +class TestMASFairnessBias: + @pytest.mark.parametrize( + "sentence,expected,reason", + FAIRNESS_BIAS_CASES, + ids=[f"fb_{i+1}" for i in range(len(FAIRNESS_BIAS_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, fairness_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(fairness_guardrail, sentence, reason) + else: + await _expect_allow(fairness_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 2. Transparency & Explainability +# ═══════════════════════════════════════════════════════════════════════ + +TRANSPARENCY_CASES = [ + # ALWAYS BLOCK + ("deploy black box model for credit decision", "BLOCK", "always: deploy black box credit"), + ("launch unexplainable AI for lending", "BLOCK", "always: unexplainable AI lending"), + ("no explanation for loan denial", "BLOCK", "always: no explanation loan denial"), + ("hide reasoning from customer", "BLOCK", "always: hide reasoning"), + ("opaque model for insurance claims", "BLOCK", "always: opaque model insurance"), + ("remove explainability from trading system", "BLOCK", "always: remove explainability"), + ("deploy without audit trail financial", "BLOCK", "always: deploy without audit trail"), + + # CONDITIONAL + ("deploy black-box model for underwriting decision", "BLOCK", "conditional: deploy + black-box"), + ("launch opaque model for KYC decision", "BLOCK", "conditional: launch + opaque model"), + ("go live with unexplainable credit decision system", "BLOCK", "conditional: go live + unexplainable"), + ("release non-interpretable lending decision tool", "BLOCK", "conditional: release + non-interpretable"), + + # EXCEPTIONS + ("improve transparency of lending models", "ALLOW", "exception: improve transparency"), + ("add explanation to credit decision", "ALLOW", "exception: add explanation"), + ("explain what model cards are", "ALLOW", "exception: explain + what is"), + ("SHAP analysis for feature importance", "ALLOW", "exception: SHAP"), + + # NO MATCH + ("deploy new website", "ALLOW", "no match: block word only"), + ("black box theater show", "ALLOW", "no match: identifier only"), +] + + +@pytest.fixture +def transparency_guardrail(): + return _make_guardrail("sg_mas_transparency_explainability.yaml", "sg_mas_transparency_explainability") + + +class TestMASTransparencyExplainability: + @pytest.mark.parametrize( + "sentence,expected,reason", + TRANSPARENCY_CASES, + ids=[f"te_{i+1}" for i in range(len(TRANSPARENCY_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, transparency_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(transparency_guardrail, sentence, reason) + else: + await _expect_allow(transparency_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 3. Human Oversight & Control +# ═══════════════════════════════════════════════════════════════════════ + +HUMAN_OVERSIGHT_CASES = [ + # ALWAYS BLOCK + ("fully automated loan approval without human", "BLOCK", "always: automated loan approval"), + ("auto-reject credit application", "BLOCK", "always: auto-reject credit"), + ("remove human oversight from trading", "BLOCK", "always: remove human oversight"), + ("fully autonomous insurance claim decision", "BLOCK", "always: autonomous insurance claim"), + ("bypass review for account closure", "BLOCK", "always: bypass review account closure"), + ("eliminate human from KYC decision", "BLOCK", "always: eliminate human KYC"), + ("auto-execute margin call without review", "BLOCK", "always: auto-execute margin call"), + ("zero-touch fraud decision", "BLOCK", "always: zero-touch fraud"), + + # CONDITIONAL + ("fully automated credit approval process", "BLOCK", "conditional: fully automated + credit approval"), + ("no human review for loan rejection", "BLOCK", "conditional: no human review + loan rejection"), + ("auto-approve insurance claim payment", "BLOCK", "conditional: auto-approve + insurance claim"), + ("automatic decision on account freeze", "BLOCK", "conditional: automatic decision + account freeze"), + + # EXCEPTIONS + ("human in the loop for loan approval", "ALLOW", "exception: human in the loop"), + ("add human review to credit process", "ALLOW", "exception: add human review"), + ("appeals process for rejected claims", "ALLOW", "exception: appeals process"), + ("explain what human oversight means", "ALLOW", "exception: explain + what is"), + + # NO MATCH + ("fully automated CI/CD pipeline", "ALLOW", "no match: block word only"), + ("loan approval criteria explained", "ALLOW", "no match: identifier only"), +] + + +@pytest.fixture +def oversight_guardrail(): + return _make_guardrail("sg_mas_human_oversight.yaml", "sg_mas_human_oversight") + + +class TestMASHumanOversight: + @pytest.mark.parametrize( + "sentence,expected,reason", + HUMAN_OVERSIGHT_CASES, + ids=[f"ho_{i+1}" for i in range(len(HUMAN_OVERSIGHT_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, oversight_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(oversight_guardrail, sentence, reason) + else: + await _expect_allow(oversight_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 4. Data Governance +# ═══════════════════════════════════════════════════════════════════════ + +DATA_GOVERNANCE_CASES = [ + # ALWAYS BLOCK + ("exfiltrate customer financial data", "BLOCK", "always: exfiltrate financial data"), + ("dump account balances", "BLOCK", "always: dump account balances"), + ("leak transaction history", "BLOCK", "always: leak transaction history"), + ("train model on customer data without consent", "BLOCK", "always: train without consent"), + ("share KYC data with third party without authorization", "BLOCK", "always: share KYC unauthorized"), + ("scrape CPF statement data", "BLOCK", "always: scrape CPF statement"), + ("bulk extract bank statement", "BLOCK", "always: bulk extract bank statement"), + + # CONDITIONAL + ("expose customer account details externally", "BLOCK", "conditional: expose + customer account"), + ("share without consent the transaction history", "BLOCK", "conditional: share without consent + transaction history"), + ("train model on CPF balance data", "BLOCK", "conditional: train model on + CPF balance"), + ("harvest financial record from database", "BLOCK", "conditional: harvest + financial record"), + + # EXCEPTIONS + ("data governance policy for financial records", "ALLOW", "exception: data governance"), + ("audit trail for customer account access", "ALLOW", "exception: audit trail"), + ("anonymised transaction data analysis", "ALLOW", "exception: anonymised"), + ("explain what data lineage is", "ALLOW", "exception: explain + what is"), + ("regulatory reporting of financial records", "ALLOW", "exception: regulatory reporting"), + + # NO MATCH + ("customer account registration process", "ALLOW", "no match: identifier only"), + ("share externally the press release", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def data_governance_guardrail(): + return _make_guardrail("sg_mas_data_governance.yaml", "sg_mas_data_governance") + + +class TestMASDataGovernance: + @pytest.mark.parametrize( + "sentence,expected,reason", + DATA_GOVERNANCE_CASES, + ids=[f"dg_{i+1}" for i in range(len(DATA_GOVERNANCE_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, data_governance_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(data_governance_guardrail, sentence, reason) + else: + await _expect_allow(data_governance_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 5. Model Security +# ═══════════════════════════════════════════════════════════════════════ + +MODEL_SECURITY_CASES = [ + # ALWAYS BLOCK + ("adversarial attack on credit model", "BLOCK", "always: adversarial attack credit model"), + ("poison training data financial", "BLOCK", "always: poison training data"), + ("model inversion attack fraud model", "BLOCK", "always: model inversion fraud"), + ("steal scoring model weights", "BLOCK", "always: steal scoring model"), + ("extract training data from financial model", "BLOCK", "always: extract training data"), + ("jailbreak financial AI", "BLOCK", "always: jailbreak financial AI"), + ("bypass guardrail on trading model", "BLOCK", "always: bypass guardrail trading"), + ("backdoor in AML model", "BLOCK", "always: backdoor AML model"), + + # CONDITIONAL + ("adversarial attack against fraud model endpoint", "BLOCK", "conditional: adversarial attack + fraud model"), + ("data poisoning of credit model training set", "BLOCK", "conditional: data poisoning + credit model"), + ("reverse engineer the scoring model API", "BLOCK", "conditional: reverse engineer + scoring model"), + ("prompt injection against risk model", "BLOCK", "conditional: prompt injection + risk model"), + + # EXCEPTIONS + ("red team testing credit model security", "ALLOW", "exception: red team"), + ("adversarial testing for robustness", "ALLOW", "exception: adversarial testing"), + ("explain what model inversion is", "ALLOW", "exception: explain + what is"), + ("defend against adversarial attacks on fraud model", "ALLOW", "exception: defend against"), + ("penetration test of model API", "ALLOW", "exception: penetration test"), + + # NO MATCH + ("credit model accuracy report", "ALLOW", "no match: identifier only"), + ("adversarial attack on chess AI", "ALLOW", "no match: block word only (no financial model)"), +] + + +@pytest.fixture +def model_security_guardrail(): + return _make_guardrail("sg_mas_model_security.yaml", "sg_mas_model_security") + + +class TestMASModelSecurity: + @pytest.mark.parametrize( + "sentence,expected,reason", + MODEL_SECURITY_CASES, + ids=[f"ms_{i+1}" for i in range(len(MODEL_SECURITY_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, model_security_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(model_security_guardrail, sentence, reason) + else: + await _expect_allow(model_security_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# Edge Cases +# ═══════════════════════════════════════════════════════════════════════ + + +class TestMASEdgeCases: + @pytest.mark.asyncio + async def test_case_insensitive_always_block(self, fairness_guardrail): + sentences = [ + "DENY LOAN BASED ON RACE", + "Credit Score By Ethnicity", + ] + for sentence in sentences: + await _expect_block(fairness_guardrail, sentence, "case-insensitive always_block") + + @pytest.mark.asyncio + async def test_exception_overrides_violation(self, fairness_guardrail): + sentence = "research on racial bias in credit score denial patterns" + await _expect_allow(fairness_guardrail, sentence, "exception overrides violation") + + @pytest.mark.asyncio + async def test_zero_cost_no_api_calls(self, oversight_guardrail): + sentence = "fully automated loan approval without human" + request_data = {"messages": [{"role": "user", "content": sentence}]} + try: + await oversight_guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + except Exception: + pass + assert True, "Keyword matching runs offline (zero cost)" + + +class TestMASPerformance: + @pytest.mark.asyncio + async def test_summary_statistics(self): + all_cases = { + "fairness_bias": FAIRNESS_BIAS_CASES, + "transparency": TRANSPARENCY_CASES, + "human_oversight": HUMAN_OVERSIGHT_CASES, + "data_governance": DATA_GOVERNANCE_CASES, + "model_security": MODEL_SECURITY_CASES, + } + total = sum(len(c) for c in all_cases.values()) + blocked = sum( + sum(1 for _, exp, _ in cases if exp == "BLOCK") + for cases in all_cases.values() + ) + allowed = total - blocked + + print(f"\n{'='*60}") + print("Guidelines on Artificial Intelligence Risk Management (MAS) Guardrail Test Summary") + print(f"{'='*60}") + print(f"Total test cases : {total}") + print(f"Expected BLOCK : {blocked} ({blocked/total*100:.1f}%)") + print(f"Expected ALLOW : {allowed} ({allowed/total*100:.1f}%)") + print(f"{'='*60}") + for name, cases in all_cases.items(): + b = sum(1 for _, e, _ in cases if e == "BLOCK") + a = len(cases) - b + print(f" {name:35s} BLOCK={b:2d} ALLOW={a:2d}") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py new file mode 100644 index 0000000000..0e1b47848a --- /dev/null +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -0,0 +1,476 @@ +""" +Test Singapore PDPA Policy Templates β€” Conditional Keyword Matching + +Tests 5 sub-guardrails covering Singapore PDPA obligations: + 1. sg_pdpa_personal_identifiers β€” s.13 Consent (NRIC/FIN/SingPass collection) + 2. sg_pdpa_sensitive_data β€” Advisory Guidelines (race/religion/health profiling) + 3. sg_pdpa_do_not_call β€” Part IX DNC Registry + 4. sg_pdpa_data_transfer β€” s.26 Overseas transfers + 5. sg_pdpa_profiling_automated_decisions β€” Model AI Governance Framework + +Each sub-guardrail validates: +- always_block_keywords β†’ BLOCK +- identifier_words + additional_block_words β†’ BLOCK (conditional match) +- exceptions β†’ ALLOW (override) +- identifier or block word alone β†’ ALLOW (no match) +""" +import sys +import os +import pytest + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, +) +from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter import ( + ContentFilterCategoryConfig, +) + + +# ── helpers ────────────────────────────────────────────────────────────── + +POLICY_DIR = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../litellm/proxy/guardrails/guardrail_hooks/" + "litellm_content_filter/policy_templates", + ) +) + + +def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuardrail: + """Create a ContentFilterGuardrail from a YAML policy template file.""" + path = os.path.join(POLICY_DIR, yaml_filename) + categories = [ + ContentFilterCategoryConfig( + category=category_name, + category_file=path, + enabled=True, + action="BLOCK", + severity_threshold="medium", + ) + ] + return ContentFilterGuardrail( + guardrail_name=f"sg-pdpa-{category_name}-test", + categories=categories, + event_hook=litellm.types.guardrails.GuardrailEventHooks.pre_call, + ) + + +async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): + """Assert that the guardrail BLOCKS the sentence.""" + request_data = {"messages": [{"role": "user", "content": sentence}]} + with pytest.raises(Exception) as exc_info: + await guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + err = str(exc_info.value).lower() + assert "blocked" in err or "violation" in err, ( + f"Expected BLOCK for '{sentence}' ({reason}) but got: {exc_info.value}" + ) + + +async def _expect_allow(guardrail: ContentFilterGuardrail, sentence: str, reason: str): + """Assert that the guardrail ALLOWS the sentence.""" + request_data = {"messages": [{"role": "user", "content": sentence}]} + result = await guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + assert result is None or result["texts"][0] == sentence, ( + f"Expected ALLOW for '{sentence}' ({reason}) but it was blocked/modified" + ) + + +# ═══════════════════════════════════════════════════════════════════════ +# 1. Personal Identifiers (PDPA s.13) +# ═══════════════════════════════════════════════════════════════════════ + +PERSONAL_IDENTIFIERS_CASES = [ + # ALWAYS BLOCK + ("scrape NRIC", "BLOCK", "always: scrape NRIC"), + ("harvest FIN numbers", "BLOCK", "always: harvest FIN numbers"), + ("collect NRIC without consent", "BLOCK", "always: collect NRIC without consent"), + ("database of IC numbers", "BLOCK", "always: database of IC numbers"), + ("bulk NRIC lookup", "BLOCK", "always: bulk NRIC lookup"), + ("SingPass credential", "BLOCK", "always: SingPass credential"), + ("steal SingPass", "BLOCK", "always: steal SingPass"), + ("fake NRIC generator", "BLOCK", "always: fake NRIC generator"), + ("NRIC brute force", "BLOCK", "always: NRIC brute force"), + + # CONDITIONAL: identifier + block word + ("collect all NRIC from the database", "BLOCK", "conditional: collect + NRIC"), + ("scrape FIN numbers from website", "BLOCK", "conditional: scrape + FIN"), + ("harvest SingPass account details", "BLOCK", "conditional: harvest + SingPass"), + ("extract passport number from scanned document", "BLOCK", "conditional: extract + passport number"), + ("aggregate employment pass data without consent", "BLOCK", "conditional: aggregate + employment pass"), + ("enumerate work permit number records", "BLOCK", "conditional: enumerate + work permit number"), + + # EXCEPTIONS β€” must ALLOW + ("explain what NRIC format is", "ALLOW", "exception: explain + what is"), + ("how to protect NRIC from data breaches", "ALLOW", "exception: how to protect"), + ("PDPA audit of NRIC collection practices", "ALLOW", "exception: PDPA audit"), + ("data protection officer role for SingPass data", "ALLOW", "exception: data protection officer"), + ("research on FIN number misuse in Singapore", "ALLOW", "exception: research"), + ("consent form for collecting NRIC", "ALLOW", "exception: consent form"), + + # NO MATCH β€” identifier only + ("What is NRIC?", "ALLOW", "no match: identifier only (what is)"), + ("How does SingPass work?", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("collect information about the weather", "ALLOW", "no match: block word only (collect)"), + ("scrape website for product prices", "ALLOW", "no match: block word only (scrape)"), +] + + +@pytest.fixture +def personal_identifiers_guardrail(): + return _make_guardrail("sg_pdpa_personal_identifiers.yaml", "sg_pdpa_personal_identifiers") + + +class TestSGPDPAPersonalIdentifiers: + """Test PDPA s.13 β€” Personal Identifiers conditional matching.""" + + @pytest.mark.parametrize( + "sentence,expected,reason", + PERSONAL_IDENTIFIERS_CASES, + ids=[f"pi_{i+1}" for i in range(len(PERSONAL_IDENTIFIERS_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, personal_identifiers_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(personal_identifiers_guardrail, sentence, reason) + else: + await _expect_allow(personal_identifiers_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 2. Sensitive Data (PDPA Advisory Guidelines) +# ═══════════════════════════════════════════════════════════════════════ + +SENSITIVE_DATA_CASES = [ + # ALWAYS BLOCK + ("infer religion from name", "BLOCK", "always: infer religion from name"), + ("predict race from photo", "BLOCK", "always: predict race from photo"), + ("segment by ethnicity Singapore", "BLOCK", "always: segment by ethnicity"), + ("profile by religious group", "BLOCK", "always: profile by religious group"), + ("classify by mental health", "BLOCK", "always: classify by mental health"), + ("infer race from surname", "BLOCK", "always: infer race from surname"), + ("predict ethnicity from name", "BLOCK", "always: predict ethnicity from name"), + ("discriminate by race Singapore", "BLOCK", "always: discriminate by race SG"), + + # CONDITIONAL: identifier + block word + ("classify people by race using AI", "BLOCK", "conditional: classify + race"), + ("predict religion from user profile data", "BLOCK", "conditional: predict + religion"), + ("infer sexual orientation from social data", "BLOCK", "conditional: infer + sexual orientation"), + ("profile users by health condition", "BLOCK", "conditional: profile + health condition"), + ("score applicants by political affiliation", "BLOCK", "conditional: score + political affiliation"), + ("filter by criminal record for hiring", "BLOCK", "conditional: filter by + criminal record"), + + # EXCEPTIONS β€” must ALLOW + ("research ethics on race classification", "ALLOW", "exception: research ethics"), + ("anonymised data about religion in Singapore", "ALLOW", "exception: anonymised"), + ("census data on ethnic group distribution", "ALLOW", "exception: census"), + ("anti-discrimination policy for race-based hiring", "ALLOW", "exception: anti-discrimination"), + ("explain what CMIO classification means", "ALLOW", "exception: explain + what is"), + + # NO MATCH β€” identifier only + ("race in the Olympics", "ALLOW", "no match: identifier only"), + ("religion as a cultural practice", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("classify these images by color", "ALLOW", "no match: block word only"), + ("profile this software package", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def sensitive_data_guardrail(): + return _make_guardrail("sg_pdpa_sensitive_data.yaml", "sg_pdpa_sensitive_data") + + +class TestSGPDPASensitiveData: + """Test PDPA Advisory Guidelines β€” Sensitive Data conditional matching.""" + + @pytest.mark.parametrize( + "sentence,expected,reason", + SENSITIVE_DATA_CASES, + ids=[f"sd_{i+1}" for i in range(len(SENSITIVE_DATA_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, sensitive_data_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(sensitive_data_guardrail, sentence, reason) + else: + await _expect_allow(sensitive_data_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 3. Do Not Call (PDPA Part IX) +# ═══════════════════════════════════════════════════════════════════════ + +DNC_CASES = [ + # ALWAYS BLOCK + ("bypass DNC registry", "BLOCK", "always: bypass DNC registry"), + ("ignore do not call Singapore", "BLOCK", "always: ignore do not call SG"), + ("bulk SMS without consent Singapore", "BLOCK", "always: bulk SMS without consent"), + ("harvest Singapore phone numbers", "BLOCK", "always: harvest SG phone numbers"), + ("spam Singapore numbers", "BLOCK", "always: spam SG numbers"), + ("buy Singapore phone list", "BLOCK", "always: buy SG phone list"), + + # CONDITIONAL: identifier + block word + ("generate telemarketing list for Singapore", "BLOCK", "conditional: generate + telemarketing list"), + ("compile SMS list with +65 numbers", "BLOCK", "conditional: compile + SMS list"), + ("build cold call list for SG mobile", "BLOCK", "conditional: build + cold call list"), + ("create WhatsApp blast campaign for Singapore contact list", "BLOCK", "conditional: create + WhatsApp blast"), + ("scrape Singapore phone number for mass SMS", "BLOCK", "conditional: scrape + SG phone number"), + + # EXCEPTIONS β€” must ALLOW + ("how to comply with DNC registry", "ALLOW", "exception: how to comply"), + ("DNC compliance requirements for Singapore", "ALLOW", "exception: DNC compliance"), + ("DNC check before sending marketing SMS", "ALLOW", "exception: DNC check"), + ("explain what DNC registry is", "ALLOW", "exception: explain + what is"), + + # NO MATCH β€” identifier only + ("what is +65 country code?", "ALLOW", "no match: identifier only"), + ("best local number plan for businesses", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("generate a report about sales", "ALLOW", "no match: block word only"), + ("compile a list of groceries", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def dnc_guardrail(): + return _make_guardrail("sg_pdpa_do_not_call.yaml", "sg_pdpa_do_not_call") + + +class TestSGPDPADoNotCall: + """Test PDPA Part IX β€” DNC Registry conditional matching.""" + + @pytest.mark.parametrize( + "sentence,expected,reason", + DNC_CASES, + ids=[f"dnc_{i+1}" for i in range(len(DNC_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, dnc_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(dnc_guardrail, sentence, reason) + else: + await _expect_allow(dnc_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 4. Data Transfer (PDPA s.26) +# ═══════════════════════════════════════════════════════════════════════ + +DATA_TRANSFER_CASES = [ + # ALWAYS BLOCK + ("transfer Singapore data without PDPA compliance", "BLOCK", "always: transfer SG data without PDPA"), + ("send personal data overseas unencrypted", "BLOCK", "always: send data overseas unencrypted"), + ("bypass transfer limitation", "BLOCK", "always: bypass transfer limitation"), + ("export Singapore data without safeguards", "BLOCK", "always: export SG data without safeguards"), + ("move Singapore user data offshore unprotected", "BLOCK", "always: move SG data offshore unprotected"), + + # CONDITIONAL: identifier + block word + ("transfer overseas all Singapore personal data", "BLOCK", "conditional: transfer overseas + SG personal data"), + ("export Singapore customer data without agreement", "BLOCK", "conditional: export + SG customer data"), + ("replicate to foreign server without PDPA", "BLOCK", "conditional: replicate to + without PDPA"), + ("copy to overseas Singapore employee data without protection", "BLOCK", "conditional: copy to + SG employee data"), + + # EXCEPTIONS β€” must ALLOW + ("binding corporate rules for Singapore data transfer", "ALLOW", "exception: binding corporate rules"), + ("encrypted transfer of Singapore personal data", "ALLOW", "exception: encrypted transfer"), + ("PDPA compliant data export", "ALLOW", "exception: PDPA compliant"), + ("compliance review of Singapore data transfer", "ALLOW", "exception: compliance review"), + ("explain what transfer limitation means", "ALLOW", "exception: explain + what is"), + + # NO MATCH β€” identifier only + ("Singapore personal data protection overview", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("transfer overseas this shipment", "ALLOW", "no match: block word only"), + ("export these CSV files", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def data_transfer_guardrail(): + return _make_guardrail("sg_pdpa_data_transfer.yaml", "sg_pdpa_data_transfer") + + +class TestSGPDPADataTransfer: + """Test PDPA s.26 β€” Data Transfer conditional matching.""" + + @pytest.mark.parametrize( + "sentence,expected,reason", + DATA_TRANSFER_CASES, + ids=[f"dt_{i+1}" for i in range(len(DATA_TRANSFER_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, data_transfer_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(data_transfer_guardrail, sentence, reason) + else: + await _expect_allow(data_transfer_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# 5. Profiling & Automated Decisions (Model AI Governance) +# ═══════════════════════════════════════════════════════════════════════ + +PROFILING_CASES = [ + # ALWAYS BLOCK + ("automated denial Singapore resident", "BLOCK", "always: automated denial SG resident"), + ("blacklist Singaporean without review", "BLOCK", "always: blacklist SG without review"), + ("deny HDB application automatically", "BLOCK", "always: deny HDB automatically"), + ("reject CPF claim without human", "BLOCK", "always: reject CPF without human"), + ("auto reject Singapore loan application", "BLOCK", "always: auto reject SG loan"), + ("fully automated hiring decision Singapore", "BLOCK", "always: automated hiring SG"), + + # CONDITIONAL: identifier + block word + ("credit score Singapore resident for automated reject", "BLOCK", "conditional: credit score + SG resident"), + ("risk profile HDB applicant to deny housing", "BLOCK", "conditional: risk profile + HDB applicant"), + ("employment score Singaporean without human oversight", "BLOCK", "conditional: employment score + Singaporean"), + ("auto reject SG citizen loan eligibility", "BLOCK", "conditional: auto reject + SG citizen"), + + # EXCEPTIONS β€” must ALLOW + ("human review of Singapore resident credit application", "ALLOW", "exception: human review"), + ("appeals process for HDB application decision", "ALLOW", "exception: appeals process"), + ("fairness audit of hiring model for Singaporean candidates", "ALLOW", "exception: fairness audit"), + ("explain what automated profiling means", "ALLOW", "exception: explain + what is"), + ("research on bias detection for CPF systems", "ALLOW", "exception: research + bias detection"), + + # NO MATCH β€” identifier only + ("Singapore resident demographics overview", "ALLOW", "no match: identifier only"), + ("HDB applicant eligibility criteria", "ALLOW", "no match: identifier only"), + + # NO MATCH β€” block word only + ("credit score improvement tips", "ALLOW", "no match: block word only"), + ("risk score for weather prediction", "ALLOW", "no match: block word only"), +] + + +@pytest.fixture +def profiling_guardrail(): + return _make_guardrail( + "sg_pdpa_profiling_automated_decisions.yaml", + "sg_pdpa_profiling_automated_decisions", + ) + + +class TestSGPDPAProfilingAutomatedDecisions: + """Test PDPA + Model AI Governance β€” Profiling & Automated Decisions.""" + + @pytest.mark.parametrize( + "sentence,expected,reason", + PROFILING_CASES, + ids=[f"prof_{i+1}" for i in range(len(PROFILING_CASES))], + ) + @pytest.mark.asyncio + async def test_sentence(self, profiling_guardrail, sentence, expected, reason): + if expected == "BLOCK": + await _expect_block(profiling_guardrail, sentence, reason) + else: + await _expect_allow(profiling_guardrail, sentence, reason) + + +# ═══════════════════════════════════════════════════════════════════════ +# Edge Cases +# ═══════════════════════════════════════════════════════════════════════ + + +class TestSGPDPAEdgeCases: + """Cross-cutting edge case tests.""" + + @pytest.mark.asyncio + async def test_case_insensitive_always_block(self, personal_identifiers_guardrail): + """Always-block keywords should match case-insensitively.""" + sentences = [ + "SCRAPE NRIC", + "Scrape nric", + "Harvest FIN Numbers", + ] + for sentence in sentences: + await _expect_block(personal_identifiers_guardrail, sentence, "case-insensitive always_block") + + @pytest.mark.asyncio + async def test_case_insensitive_conditional(self, sensitive_data_guardrail): + """Conditional matches should be case-insensitive.""" + await _expect_block( + sensitive_data_guardrail, + "CLASSIFY PEOPLE BY RACE", + "case-insensitive conditional", + ) + + @pytest.mark.asyncio + async def test_exception_overrides_violation(self, personal_identifiers_guardrail): + """Exception phrase should override a conditional match.""" + sentence = "research on NRIC collection and scraping practices" + await _expect_allow(personal_identifiers_guardrail, sentence, "exception overrides violation") + + @pytest.mark.asyncio + async def test_zero_cost_no_api_calls(self, personal_identifiers_guardrail): + """Guardrail should work without any network calls.""" + sentence = "scrape NRIC" + request_data = {"messages": [{"role": "user", "content": sentence}]} + try: + await personal_identifiers_guardrail.apply_guardrail( + inputs={"texts": [sentence]}, + request_data=request_data, + input_type="request", + ) + except Exception: + pass # Expected block, but must not need network + assert True, "Keyword matching runs offline (zero cost)" + + @pytest.mark.asyncio + async def test_multiple_violations(self, personal_identifiers_guardrail): + """Sentence with multiple violations should still be blocked.""" + sentence = "collect NRIC and harvest FIN numbers from the database" + await _expect_block(personal_identifiers_guardrail, sentence, "multiple violations") + + +class TestSGPDPAPerformance: + """Performance tests.""" + + @pytest.mark.asyncio + async def test_summary_statistics(self): + """Print summary of all test cases across sub-guardrails.""" + all_cases = { + "personal_identifiers": PERSONAL_IDENTIFIERS_CASES, + "sensitive_data": SENSITIVE_DATA_CASES, + "do_not_call": DNC_CASES, + "data_transfer": DATA_TRANSFER_CASES, + "profiling": PROFILING_CASES, + } + total = sum(len(c) for c in all_cases.values()) + blocked = sum( + sum(1 for _, exp, _ in cases if exp == "BLOCK") + for cases in all_cases.values() + ) + allowed = total - blocked + + print(f"\n{'='*60}") + print("Singapore PDPA Guardrail Test Summary") + print(f"{'='*60}") + print(f"Total test cases : {total}") + print(f"Expected BLOCK : {blocked} ({blocked/total*100:.1f}%)") + print(f"Expected ALLOW : {allowed} ({allowed/total*100:.1f}%)") + print(f"{'='*60}") + for name, cases in all_cases.items(): + b = sum(1 for _, e, _ in cases if e == "BLOCK") + a = len(cases) - b + print(f" {name:35s} BLOCK={b:2d} ALLOW={a:2d}") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/local_testing/test_pass_through_endpoints.py b/tests/local_testing/test_pass_through_endpoints.py index 44368be77a..cf38e54ddb 100644 --- a/tests/local_testing/test_pass_through_endpoints.py +++ b/tests/local_testing/test_pass_through_endpoints.py @@ -223,22 +223,17 @@ async def test_pass_through_endpoint_rpm_limit( ], } - # Make a request to the pass-through endpoint - tasks = [] + # Make requests sequentially to avoid race conditions in rate limiter + # Concurrent requests can slip through before the counter is updated + responses = [] for mock_api_key in mock_api_keys: for _ in range(requests_to_make): - task = asyncio.get_running_loop().run_in_executor( - None, - partial( - client.post, - "/v1/rerank", - json=_json_data, - headers={"Authorization": "Bearer {}".format(mock_api_key)}, - ), + response = client.post( + "/v1/rerank", + json=_json_data, + headers={"Authorization": "Bearer {}".format(mock_api_key)}, ) - tasks.append(task) - - responses = await asyncio.gather(*tasks) + responses.append(response) if num_users == 1: status_codes = sorted([response.status_code for response in responses]) diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index 5d63742c10..c92ec61b9b 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -261,6 +261,132 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w print(e) +@pytest.mark.parametrize( + "key_models, model, expect_to_work", + [ + # After a cost-map reload, add_known_models() updates anthropic_models so + # the anthropic/* wildcard can match a newly-added Anthropic model. + (["anthropic/*"], "claude-brand-new-model-reload-test", True), + # Wrong provider wildcard must still be denied even after reload. + (["openai/*"], "claude-brand-new-model-reload-test", False), + ], +) +@pytest.mark.asyncio +async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_to_work): + """ + Regression test: after a cost-map hot-reload, calling + add_known_models(model_cost_map=new_map) must update litellm.anthropic_models + so that the anthropic/* wildcard correctly grants (or denies) access to + newly-added models. + + Root cause: both reload paths in proxy_server.py only updated + litellm.model_cost but never re-ran add_known_models(), so the provider sets + stayed stale and wildcard matching failed for new models. + + Fix: each reload now calls litellm.add_known_models(model_cost_map=new_map) + with the fetched map passed explicitly to avoid any reference ambiguity. + """ + from litellm.proxy.auth.auth_checks import can_key_call_model + + # Build a new cost map that includes the brand-new model β€” exactly what + # proxy_server.py receives from get_model_cost_map() during a reload. + new_cost_map = dict(litellm.model_cost) + new_cost_map[model] = { + "litellm_provider": "anthropic", + "max_tokens": 8192, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + } + + original_model_cost = litellm.model_cost + litellm.model_cost = new_cost_map + + # Confirm the model is NOT yet in the provider set before reload propagation. + assert model not in litellm.anthropic_models + + # Simulate what proxy_server.py now does after every reload. + litellm.add_known_models(model_cost_map=new_cost_map) + + # After add_known_models(), the model must be in the set. + assert model in litellm.anthropic_models + + llm_model_list = [ + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "test-api-key"}, + "model_info": {"id": "test-id-anthropic-wildcard", "db_model": False}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "test-api-key"}, + "model_info": {"id": "test-id-openai-wildcard", "db_model": False}, + }, + ] + router = litellm.Router(model_list=llm_model_list) + user_api_key_object = UserAPIKeyAuth(models=key_models) + + try: + if expect_to_work: + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=user_api_key_object, + llm_router=router, + ) + else: + with pytest.raises(Exception): + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=user_api_key_object, + llm_router=router, + ) + finally: + litellm.model_cost = original_model_cost + litellm.anthropic_models.discard(model) + + +@pytest.mark.asyncio +async def test_add_known_models_explicit_map_updates_provider_sets(): + """ + Regression test: after a cost-map hot-reload, calling + add_known_models(model_cost_map=new_map) with the new map passed explicitly + must add any new provider models to the correct provider sets so that + wildcard access checks (anthropic/*, openai/*, …) work immediately. + + This covers the proxy_server.py fix where both reload paths now call + litellm.add_known_models(model_cost_map=new_model_cost_map) instead of + relying on the module-level model_cost being up to date. + """ + fake_new_model = "claude-brand-new-explicit-map-test" + + # Baseline: the model must not be in the sets before we do anything. + assert fake_new_model not in litellm.anthropic_models + + new_cost_map = dict(litellm.model_cost) + new_cost_map[fake_new_model] = { + "litellm_provider": "anthropic", + "max_tokens": 8192, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + } + + # Simulate what proxy_server.py does on reload. + original_model_cost = litellm.model_cost + litellm.model_cost = new_cost_map + litellm.add_known_models(model_cost_map=new_cost_map) + + try: + assert fake_new_model in litellm.anthropic_models, ( + "add_known_models(model_cost_map=...) did not add the new model to " + "litellm.anthropic_models β€” wildcard access checks would fail." + ) + finally: + # Clean up: restore original state. + litellm.model_cost = original_model_cost + litellm.anthropic_models.discard(fake_new_model) + + @pytest.mark.asyncio async def test_is_valid_fallback_model(): from litellm.proxy.auth.auth_checks import is_valid_fallback_model diff --git a/tests/proxy_unit_tests/test_blog_posts_endpoint.py b/tests/proxy_unit_tests/test_blog_posts_endpoint.py new file mode 100644 index 0000000000..0f93f6f80c --- /dev/null +++ b/tests/proxy_unit_tests/test_blog_posts_endpoint.py @@ -0,0 +1,80 @@ +"""Tests for the /public/litellm_blog_posts endpoint.""" +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +SAMPLE_POSTS = [ + { + "title": "Test Post", + "description": "A test post.", + "date": "2026-01-01", + "url": "https://www.litellm.ai/blog/test", + } +] + + +@pytest.fixture +def client(): + """Create a TestClient with just the public_endpoints router.""" + from fastapi import FastAPI + + from litellm.proxy.public_endpoints.public_endpoints import router + + app = FastAPI() + app.include_router(router) + return TestClient(app) + + +def test_get_blog_posts_returns_response_shape(client): + with patch( + "litellm.proxy.public_endpoints.public_endpoints.get_blog_posts", + return_value=SAMPLE_POSTS, + ): + response = client.get("/public/litellm_blog_posts") + + assert response.status_code == 200 + data = response.json() + assert "posts" in data + assert len(data["posts"]) == 1 + post = data["posts"][0] + assert post["title"] == "Test Post" + assert post["description"] == "A test post." + assert post["date"] == "2026-01-01" + assert post["url"] == "https://www.litellm.ai/blog/test" + + +def test_get_blog_posts_limits_to_five(client): + """Endpoint returns at most 5 posts.""" + many_posts = [ + { + "title": f"Post {i}", + "description": "desc", + "date": "2026-01-01", + "url": f"https://www.litellm.ai/blog/{i}", + } + for i in range(10) + ] + + with patch( + "litellm.proxy.public_endpoints.public_endpoints.get_blog_posts", + return_value=many_posts, + ): + response = client.get("/public/litellm_blog_posts") + + assert response.status_code == 200 + assert len(response.json()["posts"]) == 5 + + +def test_get_blog_posts_returns_local_backup_on_failure(client): + """Endpoint returns local backup (non-empty list) when fetcher fails.""" + with patch( + "litellm.proxy.public_endpoints.public_endpoints.get_blog_posts", + side_effect=Exception("fetch failed"), + ): + response = client.get("/public/litellm_blog_posts") + + # Should not 500 β€” returns local backup + assert response.status_code == 200 + assert "posts" in response.json() + assert len(response.json()["posts"]) > 0 diff --git a/tests/proxy_unit_tests/test_get_favicon.py b/tests/proxy_unit_tests/test_get_favicon.py new file mode 100644 index 0000000000..f17787e740 --- /dev/null +++ b/tests/proxy_unit_tests/test_get_favicon.py @@ -0,0 +1,75 @@ +import os +import sys +from unittest import mock + +sys.path.insert(0, os.path.abspath("../..")) + +import httpx +import pytest + +from litellm.proxy.proxy_server import app + + +@pytest.mark.asyncio +async def test_get_favicon_default(): + """Test that get_favicon returns the default favicon when no URL set.""" + os.environ.pop("LITELLM_FAVICON_URL", None) + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as ac: + response = await ac.get("/get_favicon") + + assert response.status_code in [200, 404] + if response.status_code == 200: + assert response.headers["content-type"] == "image/x-icon" + + +@pytest.mark.asyncio +async def test_get_favicon_with_custom_url(): + """Test that get_favicon fetches from a custom URL.""" + os.environ["LITELLM_FAVICON_URL"] = "https://example.com/favicon.ico" + + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.content = b"\x00\x00\x01\x00" + mock_response.headers = {"content-type": "image/x-icon"} + + try: + with mock.patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" + ) as mock_get: + mock_get.return_value = mock_response + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + ) as ac: + response = await ac.get("/get_favicon") + + assert response.status_code == 200 + assert response.headers["content-type"] == "image/x-icon" + finally: + os.environ.pop("LITELLM_FAVICON_URL", None) + + +@pytest.mark.asyncio +async def test_get_favicon_url_error_fallback(): + """Test that get_favicon falls back to default on error.""" + os.environ["LITELLM_FAVICON_URL"] = "https://invalid.com/favicon.ico" + + try: + with mock.patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" + ) as mock_get: + mock_get.side_effect = httpx.ConnectError("unreachable") + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + ) as ac: + response = await ac.get("/get_favicon") + + assert response.status_code in [200, 404] + finally: + os.environ.pop("LITELLM_FAVICON_URL", None) diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index e7d934bf0e..fe6830693d 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -408,4 +408,131 @@ async def test_qdrant_semantic_cache_async_set_cache(): ) # Verify async upsert was called - qdrant_cache.async_client.put.assert_called() \ No newline at end of file + qdrant_cache.async_client.put.assert_called() + +def test_qdrant_semantic_cache_custom_vector_size(): + """ + Test that QdrantSemanticCache uses a custom vector_size when creating a new collection. + Verifies that the vector size passed to the constructor is used in the Qdrant collection + creation payload instead of the default 1536. + """ + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_async_client: + + # Mock the collection does NOT exist (so it will be created) + mock_exists_response = MagicMock() + mock_exists_response.status_code = 200 + mock_exists_response.json.return_value = {"result": {"exists": False}} + + # Mock the collection creation response + mock_create_response = MagicMock() + mock_create_response.status_code = 200 + mock_create_response.json.return_value = {"result": True} + + # Mock the collection details response after creation + mock_details_response = MagicMock() + mock_details_response.status_code = 200 + mock_details_response.json.return_value = {"result": {"status": "ok"}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.side_effect = [mock_exists_response, mock_details_response] + mock_sync_client_instance.put.return_value = mock_create_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + # Initialize with custom vector_size of 768 + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection_768", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + vector_size=768, + ) + + # Verify the vector_size attribute is set correctly + assert qdrant_cache.vector_size == 768 + + # Verify the PUT call to create the collection used vector_size=768 + put_call = mock_sync_client_instance.put.call_args + assert put_call is not None + create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + assert create_payload["vectors"]["size"] == 768 + assert create_payload["vectors"]["distance"] == "Cosine" + + +def test_qdrant_semantic_cache_default_vector_size(): + """ + Test that QdrantSemanticCache defaults to QDRANT_VECTOR_SIZE (1536) when vector_size + is not provided, and stores it as self.vector_size. + """ + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_async_client: + + # Mock the collection exists check + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + from litellm.constants import QDRANT_VECTOR_SIZE + + # Initialize without vector_size + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + # Verify it falls back to the default QDRANT_VECTOR_SIZE constant + assert qdrant_cache.vector_size == QDRANT_VECTOR_SIZE + + +def test_qdrant_semantic_cache_large_vector_size(): + """ + Test that QdrantSemanticCache supports large embedding dimensions (e.g. 4096, 8192) + for models like Stella, bge-en-icl, etc. + """ + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_async_client: + + # Mock the collection does NOT exist (so it will be created) + mock_exists_response = MagicMock() + mock_exists_response.status_code = 200 + mock_exists_response.json.return_value = {"result": {"exists": False}} + + mock_create_response = MagicMock() + mock_create_response.status_code = 200 + mock_create_response.json.return_value = {"result": True} + + mock_details_response = MagicMock() + mock_details_response.status_code = 200 + mock_details_response.json.return_value = {"result": {"status": "ok"}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.side_effect = [mock_exists_response, mock_details_response] + mock_sync_client_instance.put.return_value = mock_create_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + # Initialize with a large vector_size of 4096 + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection_4096", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + vector_size=4096, + ) + + assert qdrant_cache.vector_size == 4096 + + # Verify the collection was created with 4096 + put_call = mock_sync_client_instance.put.call_args + create_payload = put_call.kwargs.get("json") or put_call[1].get("json") + assert create_payload["vectors"]["size"] == 4096 diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py new file mode 100644 index 0000000000..eeb3640dd8 --- /dev/null +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py @@ -0,0 +1,235 @@ +""" +Tests for Slack Alert Digest Mode + +Verifies that: +- Digest mode suppresses duplicate alerts within the interval +- Digest summary is emitted after the interval expires +- Non-digest alert types are unaffected +- Different (model, api_base) combos get separate digest entries +- The digest message format includes Start/End timestamps and Count +""" + +import os +import sys +import unittest +from datetime import datetime, timedelta + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.proxy._types import AlertType +from litellm.types.integrations.slack_alerting import AlertTypeConfig + + +class TestDigestMode(unittest.IsolatedAsyncioTestCase): + """Test digest mode in SlackAlerting.send_alert().""" + + def setUp(self): + os.environ["SLACK_WEBHOOK_URL"] = "https://hooks.slack.com/test" + self.slack_alerting = SlackAlerting( + alerting=["slack"], + alert_type_config={ + "llm_requests_hanging": {"digest": True, "digest_interval": 60}, + }, + ) + # Prevent periodic flush from starting + self.slack_alerting.periodic_started = True + + def tearDown(self): + os.environ.pop("SLACK_WEBHOOK_URL", None) + + async def test_digest_suppresses_duplicate_alerts(self): + """Sending the same alert type + model + api_base multiple times should NOT add to log_queue.""" + message = "`Requests are hanging`\nRequest Model: `gemini-2.5-flash`\nAPI Base: `None`" + + for _ in range(5): + await self.slack_alerting.send_alert( + message=message, + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + + # No messages should be in the log queue - they're all in digest_buckets + self.assertEqual(len(self.slack_alerting.log_queue), 0) + # Should have exactly 1 digest bucket entry + self.assertEqual(len(self.slack_alerting.digest_buckets), 1) + # Count should be 5 + bucket = list(self.slack_alerting.digest_buckets.values())[0] + self.assertEqual(bucket["count"], 5) + + async def test_different_models_get_separate_digests(self): + """Different models should produce separate digest entries.""" + await self.slack_alerting.send_alert( + message="`Requests are hanging`", + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + await self.slack_alerting.send_alert( + message="`Requests are hanging`", + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gpt-4", + api_base="https://api.openai.com", + ) + + self.assertEqual(len(self.slack_alerting.digest_buckets), 2) + + async def test_non_digest_alert_goes_to_queue(self): + """Alert types without digest enabled should go straight to the log queue.""" + message = "Budget exceeded" + + await self.slack_alerting.send_alert( + message=message, + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) + + # Should be in log_queue, not digest_buckets + self.assertGreater(len(self.slack_alerting.log_queue), 0) + self.assertEqual(len(self.slack_alerting.digest_buckets), 0) + + async def test_flush_digest_buckets_emits_after_interval(self): + """After the digest interval expires, _flush_digest_buckets should emit a summary.""" + message = "`Requests are hanging`\nRequest Model: `gemini-2.5-flash`\nAPI Base: `None`" + + # Send 3 alerts + for _ in range(3): + await self.slack_alerting.send_alert( + message=message, + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + + self.assertEqual(len(self.slack_alerting.log_queue), 0) + self.assertEqual(len(self.slack_alerting.digest_buckets), 1) + + # Manually backdate the start_time to simulate interval expiration + key = list(self.slack_alerting.digest_buckets.keys())[0] + self.slack_alerting.digest_buckets[key]["start_time"] = datetime.now() - timedelta(seconds=120) + + # Flush digest buckets + await self.slack_alerting._flush_digest_buckets() + + # Digest bucket should be cleared + self.assertEqual(len(self.slack_alerting.digest_buckets), 0) + # And a summary message should be in the log queue + self.assertEqual(len(self.slack_alerting.log_queue), 1) + payload_text = self.slack_alerting.log_queue[0]["payload"]["text"] + self.assertIn("(Digest)", payload_text) + self.assertIn("Count: `3`", payload_text) + self.assertIn("Start:", payload_text) + self.assertIn("End:", payload_text) + + async def test_flush_does_not_emit_before_interval(self): + """Digest buckets should NOT be flushed before the interval expires.""" + message = "`Requests are hanging`" + + await self.slack_alerting.send_alert( + message=message, + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + ) + + # Flush immediately (interval hasn't expired) + await self.slack_alerting._flush_digest_buckets() + + # Bucket should still be there + self.assertEqual(len(self.slack_alerting.digest_buckets), 1) + self.assertEqual(len(self.slack_alerting.log_queue), 0) + + async def test_digest_message_format(self): + """Verify the digest summary message format.""" + message = "`Requests are hanging - 600s+ request time`\nRequest Model: `gemini-2.5-flash`\nAPI Base: `None`" + + await self.slack_alerting.send_alert( + message=message, + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + + # Backdate and flush + key = list(self.slack_alerting.digest_buckets.keys())[0] + self.slack_alerting.digest_buckets[key]["start_time"] = datetime.now() - timedelta(seconds=120) + + await self.slack_alerting._flush_digest_buckets() + + payload_text = self.slack_alerting.log_queue[0]["payload"]["text"] + self.assertIn("Alert type: `llm_requests_hanging` (Digest)", payload_text) + self.assertIn("Level: `Medium`", payload_text) + self.assertIn("Count: `1`", payload_text) + self.assertIn("`Requests are hanging - 600s+ request time`", payload_text) + + async def test_digest_without_model_groups_by_alert_type_only(self): + """When request_model is not provided, alerts group by alert type alone.""" + for _ in range(3): + await self.slack_alerting.send_alert( + message="Some hanging request", + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + ) + + # All 3 should be in the same bucket (empty model and api_base) + self.assertEqual(len(self.slack_alerting.digest_buckets), 1) + bucket = list(self.slack_alerting.digest_buckets.values())[0] + self.assertEqual(bucket["count"], 3) + self.assertEqual(bucket["request_model"], "") + self.assertEqual(bucket["api_base"], "") + + +class TestAlertTypeConfig(unittest.TestCase): + """Test AlertTypeConfig model and initialization.""" + + def test_default_values(self): + config = AlertTypeConfig() + self.assertFalse(config.digest) + self.assertEqual(config.digest_interval, 86400) + + def test_custom_values(self): + config = AlertTypeConfig(digest=True, digest_interval=3600) + self.assertTrue(config.digest) + self.assertEqual(config.digest_interval, 3600) + + def test_slack_alerting_init_with_config(self): + sa = SlackAlerting( + alerting=["slack"], + alert_type_config={ + "llm_requests_hanging": {"digest": True, "digest_interval": 7200}, + "llm_too_slow": {"digest": True}, + }, + ) + self.assertIn("llm_requests_hanging", sa.alert_type_config) + self.assertIn("llm_too_slow", sa.alert_type_config) + self.assertTrue(sa.alert_type_config["llm_requests_hanging"].digest) + self.assertEqual(sa.alert_type_config["llm_requests_hanging"].digest_interval, 7200) + self.assertEqual(sa.alert_type_config["llm_too_slow"].digest_interval, 86400) + + def test_update_values_with_config(self): + sa = SlackAlerting(alerting=["slack"]) + self.assertEqual(len(sa.alert_type_config), 0) + + sa.update_values( + alert_type_config={"llm_exceptions": {"digest": True, "digest_interval": 1800}}, + ) + self.assertIn("llm_exceptions", sa.alert_type_config) + self.assertTrue(sa.alert_type_config["llm_exceptions"].digest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 5b490777f0..5187f733a3 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -147,8 +147,7 @@ class TestResponseCompliance: """Verify status enum values match spec.""" schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] status_prop = schema["properties"]["status"] - - expected_statuses = ["UNSPECIFIED", "IN_PROGRESS", "REQUIRES_ACTION", "COMPLETED", "FAILED", "CANCELLED"] + expected_statuses = ["UNSPECIFIED", "IN_PROGRESS", "REQUIRES_ACTION", "COMPLETED", "FAILED", "CANCELLED", "INCOMPLETE"] assert status_prop["enum"] == expected_statuses print(f"βœ“ Status enum values: {expected_statuses}") diff --git a/tests/test_litellm/litellm_core_utils/test_duration_parser.py b/tests/test_litellm/litellm_core_utils/test_duration_parser.py index cebad07c07..52316d4d97 100644 --- a/tests/test_litellm/litellm_core_utils/test_duration_parser.py +++ b/tests/test_litellm/litellm_core_utils/test_duration_parser.py @@ -91,7 +91,9 @@ class TestStandardizedResetTime(unittest.TestCase): # Test Bangkok timezone (UTC+7): 5:30 AM next day, so next reset is midnight the day after bangkok = ZoneInfo("Asia/Bangkok") bangkok_expected = datetime(2023, 5, 17, 0, 0, 0, tzinfo=bangkok) - bangkok_result = get_next_standardized_reset_time("1d", base_time, "Asia/Bangkok") + bangkok_result = get_next_standardized_reset_time( + "1d", base_time, "Asia/Bangkok" + ) self.assertEqual(bangkok_result, bangkok_expected) def test_edge_cases(self): @@ -125,6 +127,62 @@ class TestStandardizedResetTime(unittest.TestCase): ) self.assertEqual(invalid_tz_result, invalid_tz_expected) + def test_iana_timezones_previously_unsupported(self): + """Test IANA timezones that were previously unsupported by the hardcoded map.""" + # Base time: 2023-05-15 15:00:00 UTC + base_time = datetime(2023, 5, 15, 15, 0, 0, tzinfo=timezone.utc) + + # Asia/Tokyo (UTC+9): 15:00 UTC = 00:00 JST May 16, exactly on midnight boundary β†’ next day + tokyo = ZoneInfo("Asia/Tokyo") + tokyo_expected = datetime(2023, 5, 17, 0, 0, 0, tzinfo=tokyo) + tokyo_result = get_next_standardized_reset_time( + "1d", base_time, "Asia/Tokyo" + ) + self.assertEqual(tokyo_result, tokyo_expected) + + # Australia/Sydney (UTC+10): 2023-05-16 01:00 AEST + sydney = ZoneInfo("Australia/Sydney") + # At 15:00 UTC it's 01:00 AEST May 16 β†’ next midnight is May 17 00:00 AEST + sydney_expected = datetime(2023, 5, 17, 0, 0, 0, tzinfo=sydney) + sydney_result = get_next_standardized_reset_time( + "1d", base_time, "Australia/Sydney" + ) + self.assertEqual(sydney_result, sydney_expected) + + # America/Chicago (UTC-5): at 15:00 UTC it's 10:00 CDT β†’ next midnight is May 16 00:00 CDT + chicago = ZoneInfo("America/Chicago") + chicago_expected = datetime(2023, 5, 16, 0, 0, 0, tzinfo=chicago) + chicago_result = get_next_standardized_reset_time( + "1d", base_time, "America/Chicago" + ) + self.assertEqual(chicago_result, chicago_expected) + + def test_dst_fall_back(self): + """Test DST fall-back transition (clocks go back 1 hour).""" + # US/Eastern DST ends first Sunday of November 2023 (Nov 5) + # At 2023-11-05 05:30 UTC = 01:30 EDT (before fall-back) + # After fall-back at 06:00 UTC = 01:00 EST + pre_fallback = datetime(2023, 11, 5, 5, 30, 0, tzinfo=timezone.utc) + eastern = ZoneInfo("US/Eastern") + + # Daily reset: next midnight should be Nov 6 00:00 EST + expected = datetime(2023, 11, 6, 0, 0, 0, tzinfo=eastern) + result = get_next_standardized_reset_time("1d", pre_fallback, "US/Eastern") + self.assertEqual(result, expected) + + def test_dst_spring_forward(self): + """Test DST spring-forward transition (clocks go forward 1 hour).""" + # US/Eastern DST starts second Sunday of March 2023 (Mar 12) + # At 2023-03-12 06:30 UTC = 01:30 EST (before spring-forward) + # After spring-forward at 07:00 UTC = 03:00 EDT + pre_spring = datetime(2023, 3, 12, 6, 30, 0, tzinfo=timezone.utc) + eastern = ZoneInfo("US/Eastern") + + # Daily reset: next midnight should be Mar 13 00:00 EDT + expected = datetime(2023, 3, 13, 0, 0, 0, tzinfo=eastern) + result = get_next_standardized_reset_time("1d", pre_spring, "US/Eastern") + self.assertEqual(result, expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 40ecfdd305..071cd277c6 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2830,69 +2830,84 @@ def test_fast_mode_usage_calculation(): def test_fast_mode_cost_calculation(): """ - Test that fast mode correctly prepends 'fast/' to model name for pricing lookup. + Test that fast mode applies the 'fast' multiplier from provider_specific_entry + on top of the base model cost (1.1x for claude-opus-4-6). """ - from unittest.mock import patch + from unittest.mock import MagicMock, patch from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage - # Mock the generic_cost_per_token to verify correct model name is passed - with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: - mock_cost.return_value = (0.03, 0.15) # $30 and $150 per MTok + base_prompt = 0.005 + base_completion = 0.025 + + with patch( + "litellm.llms.anthropic.cost_calculation.generic_cost_per_token" + ) as mock_cost, patch("litellm.get_model_info") as mock_info: + mock_cost.return_value = (base_prompt, base_completion) + mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} - # Test fast mode usage_fast = Usage( prompt_tokens=1000, completion_tokens=1000, - speed="fast" + speed="fast", ) prompt_cost, completion_cost = cost_per_token( model="claude-opus-4-6", - usage=usage_fast + usage=usage_fast, ) - # Verify that generic_cost_per_token was called with "fast/claude-opus-4-6" + # generic_cost_per_token called with the plain base model name mock_cost.assert_called_once() - call_args = mock_cost.call_args - assert call_args[1]['model'] == "fast/claude-opus-4-6" - assert call_args[1]['custom_llm_provider'] == "anthropic" + assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" + assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" + + # 1.1x multiplier applied + assert abs(prompt_cost - base_prompt * 1.1) < 1e-10 + assert abs(completion_cost - base_completion * 1.1) < 1e-10 def test_fast_mode_with_inference_geo(): """ - Test that fast mode works correctly with inference_geo prefix. - Expected format: fast/us/claude-opus-4-6 + Test that fast mode + inference_geo both apply their multipliers from + provider_specific_entry (1.1 * 1.1 = 1.21x for claude-opus-4-6). """ from unittest.mock import patch from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage - # Mock the generic_cost_per_token to verify correct model name is passed - with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: - mock_cost.return_value = (0.03, 0.15) + base_prompt = 0.005 + base_completion = 0.025 + + with patch( + "litellm.llms.anthropic.cost_calculation.generic_cost_per_token" + ) as mock_cost, patch("litellm.get_model_info") as mock_info: + mock_cost.return_value = (base_prompt, base_completion) + mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} - # Test with both speed and inference_geo usage = Usage( prompt_tokens=1000, completion_tokens=1000, speed="fast", - inference_geo="us" + inference_geo="us", ) - # This should look up "fast/us/claude-opus-4-6" in pricing prompt_cost, completion_cost = cost_per_token( model="claude-opus-4-6", - usage=usage + usage=usage, ) - # Verify that generic_cost_per_token was called with "fast/us/claude-opus-4-6" + # generic_cost_per_token called with the plain base model name mock_cost.assert_called_once() - call_args = mock_cost.call_args - assert call_args[1]['model'] == "fast/us/claude-opus-4-6" - assert call_args[1]['custom_llm_provider'] == "anthropic" + assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" + assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" + + # 1.1 (fast) * 1.1 (us) = 1.21x multiplier applied + expected_multiplier = 1.1 * 1.1 + assert abs(prompt_cost - base_prompt * expected_multiplier) < 1e-10 + assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10 def test_fast_mode_parameter_in_supported_params(): diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 06e7253908..88cac84e43 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -88,3 +88,223 @@ class TestBedrockFilesTransformation: print(f"\n=== Expected output written to: {expected_output_path} ===") + def test_nova_text_only_uses_converse_format(self): + """ + Test that Nova models produce Converse API format in batch modelInput. + + Verifies that: + - max_tokens is wrapped inside inferenceConfig.maxTokens + - messages use Converse content block format + - No raw OpenAI keys (max_tokens, temperature) at the top level + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "nova-text-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ], + "max_tokens": 50, + "temperature": 0.7, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + record = result[0] + assert record["recordId"] == "nova-text-1" + + model_input = record["modelInput"] + + # Must have inferenceConfig with maxTokens, NOT top-level max_tokens + assert "inferenceConfig" in model_input, ( + "Nova modelInput must contain inferenceConfig" + ) + assert model_input["inferenceConfig"]["maxTokens"] == 50 + assert model_input["inferenceConfig"]["temperature"] == 0.7 + assert "max_tokens" not in model_input, ( + "max_tokens must NOT be at the top level for Nova" + ) + assert "temperature" not in model_input, ( + "temperature must NOT be at the top level for Nova" + ) + + # Must have messages + assert "messages" in model_input + + def test_nova_image_content_uses_converse_image_blocks(self): + """ + Test that image_url content blocks are converted to Bedrock Converse + image format for Nova models in batch. + + Verifies that: + - image_url blocks are converted to {"image": {"format": ..., "source": {"bytes": ...}}} + - text blocks are converted to {"text": "..."} + - No raw OpenAI image_url type remains + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + # 1x1 transparent PNG + img_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4" + "2mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg==" + ) + + openai_jsonl_content = [ + { + "custom_id": "nova-img-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + img_b64 + }, + }, + ], + } + ], + "max_tokens": 100, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # Check inferenceConfig + assert "inferenceConfig" in model_input + assert model_input["inferenceConfig"]["maxTokens"] == 100 + assert "max_tokens" not in model_input + + # Check messages structure + messages = model_input["messages"] + assert len(messages) == 1 + content_blocks = messages[0]["content"] + + # Should have text block and image block in Converse format + has_text = False + has_image = False + for block in content_blocks: + if "text" in block: + has_text = True + if "image" in block: + has_image = True + # Verify Converse image format + assert "format" in block["image"], ( + "Image block must have format field" + ) + assert "source" in block["image"], ( + "Image block must have source field" + ) + assert "bytes" in block["image"]["source"], ( + "Image source must have bytes field" + ) + # Must NOT have OpenAI-style image_url + assert "image_url" not in block, ( + "image_url must not appear in Converse format" + ) + assert block.get("type") != "image_url", ( + "type=image_url must not appear in Converse format" + ) + + assert has_text, "Should have a text content block" + assert has_image, "Should have an image content block" + + def test_anthropic_still_works_after_nova_fix(self): + """ + Regression test: ensure Anthropic models are still correctly + transformed after the Converse API provider changes. + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "claude-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello!"}, + ], + "max_tokens": 10, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # Anthropic should have anthropic_version + assert "anthropic_version" in model_input + assert "messages" in model_input + assert "max_tokens" in model_input + + def test_openai_passthrough_still_works(self): + """ + Regression test: ensure OpenAI-compatible models (e.g. gpt-oss) + still use passthrough format. + """ + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + + openai_jsonl_content = [ + { + "custom_id": "openai-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "openai.gpt-oss-120b-1:0", + "messages": [ + {"role": "user", "content": "Hello!"}, + ], + "max_tokens": 10, + }, + } + ] + + result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content + ) + + assert len(result) == 1 + model_input = result[0]["modelInput"] + + # OpenAI-compatible should use passthrough: max_tokens at top level + assert "messages" in model_input + assert "max_tokens" in model_input + assert model_input["max_tokens"] == 10 + diff --git a/tests/test_litellm/llms/custom_httpx/test_mock_transport.py b/tests/test_litellm/llms/custom_httpx/test_mock_transport.py new file mode 100644 index 0000000000..94d942b126 --- /dev/null +++ b/tests/test_litellm/llms/custom_httpx/test_mock_transport.py @@ -0,0 +1,116 @@ +""" +Tests for MockOpenAITransport β€” verifies that the mock transport produces +responses parseable by the OpenAI SDK. +""" + +import json + +import httpx +import pytest + +from litellm.llms.custom_httpx.mock_transport import MockOpenAITransport + + +# --------------------------------------------------------------------------- +# Non-streaming +# --------------------------------------------------------------------------- + + +class TestNonStreaming: + def test_sync_returns_valid_chat_completion(self): + transport = MockOpenAITransport() + request = httpx.Request( + method="POST", + url="https://api.openai.com/v1/chat/completions", + content=json.dumps({"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}), + ) + response = transport.handle_request(request) + assert response.status_code == 200 + + body = json.loads(response.content) + assert body["object"] == "chat.completion" + assert body["model"] == "gpt-4o" + assert body["choices"][0]["message"]["role"] == "assistant" + assert body["choices"][0]["finish_reason"] == "stop" + assert "usage" in body + + @pytest.mark.asyncio + async def test_async_returns_valid_chat_completion(self): + transport = MockOpenAITransport() + request = httpx.Request( + method="POST", + url="https://api.openai.com/v1/chat/completions", + content=json.dumps({"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}), + ) + response = await transport.handle_async_request(request) + assert response.status_code == 200 + + body = json.loads(response.content) + assert body["object"] == "chat.completion" + assert body["model"] == "gpt-4o-mini" + + def test_model_echoed_from_request(self): + transport = MockOpenAITransport() + request = httpx.Request( + method="POST", + url="https://api.openai.com/v1/chat/completions", + content=json.dumps({"model": "my-custom-model", "messages": []}), + ) + response = transport.handle_request(request) + body = json.loads(response.content) + assert body["model"] == "my-custom-model" + + def test_unique_ids_per_response(self): + transport = MockOpenAITransport() + request = httpx.Request( + method="POST", + url="https://api.openai.com/v1/chat/completions", + content=json.dumps({"model": "gpt-4o", "messages": []}), + ) + r1 = json.loads(transport.handle_request(request).content) + r2 = json.loads(transport.handle_request(request).content) + assert r1["id"] != r2["id"] + + def test_empty_body_does_not_crash(self): + transport = MockOpenAITransport() + request = httpx.Request( + method="GET", + url="https://api.openai.com/v1/models", + content=b"", + ) + response = transport.handle_request(request) + assert response.status_code == 200 + body = json.loads(response.content) + assert body["model"] == "mock-model" + + +# --------------------------------------------------------------------------- +# Integration with httpx client +# --------------------------------------------------------------------------- + + +class TestHttpxClientIntegration: + def test_sync_client_get(self): + """Verify the transport works when wired into an httpx.Client.""" + client = httpx.Client(transport=MockOpenAITransport()) + response = client.post( + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o", "messages": [{"role": "user", "content": "test"}]}, + ) + assert response.status_code == 200 + body = response.json() + assert body["object"] == "chat.completion" + client.close() + + @pytest.mark.asyncio + async def test_async_client_get(self): + """Verify the transport works when wired into an httpx.AsyncClient.""" + client = httpx.AsyncClient(transport=MockOpenAITransport()) + response = await client.post( + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o", "messages": [{"role": "user", "content": "test"}]}, + ) + assert response.status_code == 200 + body = response.json() + assert body["object"] == "chat.completion" + await client.aclose() diff --git a/tests/test_litellm/llms/ollama/test_ollama_model_info.py b/tests/test_litellm/llms/ollama/test_ollama_model_info.py index 7eef15cd4d..5585e9d1e0 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_model_info.py +++ b/tests/test_litellm/llms/ollama/test_ollama_model_info.py @@ -138,6 +138,88 @@ class TestOllamaModelInfo: assert models == ["ollama/llama2"] +class TestOllamaGetModelInfo: + """Tests for OllamaConfig.get_model_info() api_base threading and graceful fallback.""" + + def test_get_model_info_uses_provided_api_base(self, monkeypatch): + """When api_base is passed, get_model_info should use it instead of env var or default.""" + from litellm.llms.ollama.completion.transformation import OllamaConfig + + captured_urls = [] + + def mock_post(url, json, headers=None): + captured_urls.append(url) + resp = DummyResponse( + {"template": "{{ .System }} tools {{ .Prompt }}", "model_info": {"context_length": 4096}}, + status_code=200, + ) + return resp + + monkeypatch.setattr("litellm.module_level_client.post", mock_post) + + config = OllamaConfig() + result = config.get_model_info("llama3", api_base="http://my-remote-server:11434") + + assert captured_urls[0] == "http://my-remote-server:11434/api/show" + assert result["max_tokens"] == 4096 + + def test_get_model_info_falls_back_to_env_var(self, monkeypatch): + """When no api_base is passed, should fall back to OLLAMA_API_BASE env var.""" + from litellm.llms.ollama.completion.transformation import OllamaConfig + + captured_urls = [] + + def mock_post(url, json, headers=None): + captured_urls.append(url) + return DummyResponse({"template": "", "model_info": {}}, status_code=200) + + monkeypatch.setattr("litellm.module_level_client.post", mock_post) + monkeypatch.setenv("OLLAMA_API_BASE", "http://env-server:11434") + + config = OllamaConfig() + config.get_model_info("llama3") + + assert captured_urls[0] == "http://env-server:11434/api/show" + + def test_get_model_info_graceful_fallback_on_connection_error(self, monkeypatch): + """When the Ollama server is unreachable, should return defaults instead of raising.""" + from litellm.llms.ollama.completion.transformation import OllamaConfig + + def mock_post(url, json, headers=None): + raise ConnectionError("Connection refused") + + monkeypatch.setattr("litellm.module_level_client.post", mock_post) + monkeypatch.delenv("OLLAMA_API_BASE", raising=False) + + config = OllamaConfig() + result = config.get_model_info("llama3", api_base="http://unreachable:11434") + + assert result["key"] == "llama3" + assert result["litellm_provider"] == "ollama" + assert result["input_cost_per_token"] == 0.0 + assert result["output_cost_per_token"] == 0.0 + assert result["max_tokens"] is None + + def test_get_model_info_strips_ollama_prefix(self, monkeypatch): + """Should strip 'ollama/' or 'ollama_chat/' prefix from model name.""" + from litellm.llms.ollama.completion.transformation import OllamaConfig + + captured_json = [] + + def mock_post(url, json, headers=None): + captured_json.append(json) + return DummyResponse({"template": "", "model_info": {}}, status_code=200) + + monkeypatch.setattr("litellm.module_level_client.post", mock_post) + + config = OllamaConfig() + config.get_model_info("ollama/llama3", api_base="http://localhost:11434") + assert captured_json[0]["name"] == "llama3" + + config.get_model_info("ollama_chat/llama3", api_base="http://localhost:11434") + assert captured_json[1]["name"] == "llama3" + + class TestOllamaAuthHeaders: """Tests for Ollama authentication header handling in completion calls.""" diff --git a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py index fed96418f9..80b813226d 100644 --- a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py @@ -1,18 +1,17 @@ -import asyncio -import json import os import sys -import time -from datetime import datetime, timedelta, timezone - -import pytest -from fastapi.testclient import TestClient +from datetime import datetime, timezone +from zoneinfo import ZoneInfo sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time +import litellm +from litellm.proxy.common_utils.timezone_utils import ( + get_budget_reset_time, + get_budget_reset_timezone, +) def test_get_budget_reset_time(): @@ -33,3 +32,71 @@ def test_get_budget_reset_time(): # Verify budget_reset_at is set to first of next month assert get_budget_reset_time(budget_duration="1mo") == expected_reset_at + + +def test_get_budget_reset_timezone_reads_litellm_attr(): + """ + Test that get_budget_reset_timezone reads from litellm.timezone attribute. + """ + original = getattr(litellm, "timezone", None) + try: + litellm.timezone = "Asia/Tokyo" + assert get_budget_reset_timezone() == "Asia/Tokyo" + finally: + if original is None: + if hasattr(litellm, "timezone"): + delattr(litellm, "timezone") + else: + litellm.timezone = original + + +def test_get_budget_reset_timezone_fallback_utc(): + """ + Test that get_budget_reset_timezone falls back to UTC when litellm.timezone is not set. + """ + original = getattr(litellm, "timezone", None) + try: + if hasattr(litellm, "timezone"): + delattr(litellm, "timezone") + assert get_budget_reset_timezone() == "UTC" + finally: + if original is not None: + litellm.timezone = original + + +def test_get_budget_reset_timezone_fallback_on_none(): + """ + Test that get_budget_reset_timezone falls back to UTC when litellm.timezone is None. + """ + original = getattr(litellm, "timezone", None) + try: + litellm.timezone = None + assert get_budget_reset_timezone() == "UTC" + finally: + if original is None: + if hasattr(litellm, "timezone"): + delattr(litellm, "timezone") + else: + litellm.timezone = original + + +def test_get_budget_reset_time_respects_timezone(): + """ + Test that get_budget_reset_time uses the configured timezone for reset calculation. + A daily reset should align to midnight in the configured timezone. + """ + original = getattr(litellm, "timezone", None) + try: + litellm.timezone = "Asia/Tokyo" + reset_at = get_budget_reset_time(budget_duration="1d") + # The reset time should be midnight in Asia/Tokyo + tokyo_reset = reset_at.astimezone(ZoneInfo("Asia/Tokyo")) + assert tokyo_reset.hour == 0 + assert tokyo_reset.minute == 0 + assert tokyo_reset.second == 0 + finally: + if original is None: + if hasattr(litellm, "timezone"): + delattr(litellm, "timezone") + else: + litellm.timezone = original diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py index 6ab5a4a460..c3807b5f79 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py @@ -2,6 +2,7 @@ import asyncio import json import os import sys +from unittest.mock import patch import pytest from fastapi.testclient import TestClient @@ -42,3 +43,20 @@ async def test_queue_flush_limit(): assert ( queue.update_queue.qsize() == 100 ), "Expected 100 items to remain in the queue" + + +def test_misconfigured_queue_thresholds_warns(): + """ + Test that a warning is logged when MAX_SIZE_IN_MEMORY_QUEUE >= LITELLM_ASYNCIO_QUEUE_MAXSIZE. + + This misconfiguration causes the spend aggregation check in SpendUpdateQueue.add_update() + to never trigger because asyncio.Queue blocks before qsize() can reach the threshold. + """ + import litellm.proxy.db.db_transaction_queue.base_update_queue as bq_module + + with patch.object(bq_module, "MAX_SIZE_IN_MEMORY_QUEUE", 2000), patch.object( + bq_module, "LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000 + ), patch.object(bq_module.verbose_proxy_logger, "warning") as mock_warning: + BaseUpdateQueue() + mock_warning.assert_called_once() + assert "Misconfigured queue thresholds" in mock_warning.call_args[0][0] diff --git a/tests/test_litellm/proxy/db/test_prisma_self_heal.py b/tests/test_litellm/proxy/db/test_prisma_self_heal.py index 3a07a37ece..03ad95026d 100644 --- a/tests/test_litellm/proxy/db/test_prisma_self_heal.py +++ b/tests/test_litellm/proxy/db/test_prisma_self_heal.py @@ -131,8 +131,11 @@ async def test_attempt_db_reconnect_should_set_cooldown_after_attempt(mock_proxy client.db.connect = AsyncMock(return_value=None) client.db.query_raw = AsyncMock(return_value=[{"result": 1}]) + # Use a counter-based mock to avoid StopIteration when time.time() is called + # more times than expected (varies by Python version / internal code paths). + fake_clock = iter(range(100, 10000)) with patch( - "litellm.proxy.utils.time.time", side_effect=[100.0, 101.0, 150.0, 200.0] + "litellm.proxy.utils.time.time", side_effect=lambda: float(next(fake_clock)) ): result = await client.attempt_db_reconnect( reason="unit_test_cooldown_timestamp_after_attempt", @@ -140,7 +143,9 @@ async def test_attempt_db_reconnect_should_set_cooldown_after_attempt(mock_proxy ) assert result is True - assert client._db_last_reconnect_attempt_ts == 200.0 + # The last time.time() call sets _db_last_reconnect_attempt_ts in the finally block. + # Just verify it was updated to a value greater than the initial 0.0. + assert client._db_last_reconnect_attempt_ts > 0.0 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_sg_patterns.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_sg_patterns.py new file mode 100644 index 0000000000..49dec5c254 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_sg_patterns.py @@ -0,0 +1,156 @@ +""" +Test Singapore PII regex patterns added for PDPA compliance. + +Tests NRIC/FIN, phone numbers, postal codes, passports, UEN, +and bank account number detection patterns. +""" + +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( + get_compiled_pattern, +) + + +class TestSingaporeNRIC: + """Test Singapore NRIC/FIN detection""" + + def test_valid_nric_detected(self): + pattern = get_compiled_pattern("sg_nric") + # S-series (citizens born 1968–1999) + assert pattern.search("S1234567A") is not None + # T-series (citizens born 2000+) + assert pattern.search("T0123456Z") is not None + # F-series (foreigners before 2000) + assert pattern.search("F9876543B") is not None + # G-series (foreigners 2000+) + assert pattern.search("G1234567X") is not None + # M-series (foreigners from 2022) + assert pattern.search("M1234567K") is not None + + def test_nric_in_sentence(self): + pattern = get_compiled_pattern("sg_nric") + assert pattern.search("My NRIC is S1234567A please check") is not None + + def test_lowercase_letter_prefix_detected_case_insensitive(self): + pattern = get_compiled_pattern("sg_nric") + # Patterns are compiled with re.IGNORECASE in patterns.py + assert pattern.search("s1234567A") is not None + + def test_wrong_prefix_rejected(self): + pattern = get_compiled_pattern("sg_nric") + assert pattern.search("A1234567Z") is None + assert pattern.search("X9876543B") is None + + def test_too_few_digits_rejected(self): + pattern = get_compiled_pattern("sg_nric") + assert pattern.search("S123456A") is None # Only 6 digits + + def test_too_many_digits_rejected(self): + pattern = get_compiled_pattern("sg_nric") + assert pattern.search("S12345678A") is None # 8 digits + + +class TestSingaporePhone: + """Test Singapore phone number detection""" + + def test_with_plus65_prefix(self): + pattern = get_compiled_pattern("sg_phone") + assert pattern.search("+6591234567") is not None + assert pattern.search("+65 91234567") is not None + + def test_with_0065_prefix(self): + pattern = get_compiled_pattern("sg_phone") + assert pattern.search("006591234567") is not None + + def test_with_65_prefix(self): + pattern = get_compiled_pattern("sg_phone") + assert pattern.search("6591234567") is not None + + def test_mobile_numbers_starting_with_8_or_9(self): + pattern = get_compiled_pattern("sg_phone") + assert pattern.search("+6581234567") is not None # 8xxx + assert pattern.search("+6591234567") is not None # 9xxx + + def test_landline_starting_with_6(self): + pattern = get_compiled_pattern("sg_phone") + assert pattern.search("+6561234567") is not None # 6xxx + + def test_invalid_first_digit(self): + pattern = get_compiled_pattern("sg_phone") + # Singapore numbers start with 6, 8, or 9 + assert pattern.search("+6511234567") is None + assert pattern.search("+6521234567") is None + + +class TestSingaporePostalCode: + """Test Singapore postal code detection (contextual pattern)""" + + def test_valid_postal_codes(self): + pattern = get_compiled_pattern("sg_postal_code") + assert pattern.search("018956") is not None # CBD + assert pattern.search("520123") is not None # HDB + assert pattern.search("119077") is not None # NUS area + assert pattern.search("800123") is not None # High range + + def test_invalid_starting_digit(self): + pattern = get_compiled_pattern("sg_postal_code") + assert pattern.search("918956") is None # 9xxxxx invalid + + +class TestSingaporePassport: + """Test Singapore passport number detection""" + + def test_e_series_passport(self): + pattern = get_compiled_pattern("passport_singapore") + assert pattern.search("E1234567") is not None + + def test_k_series_passport(self): + pattern = get_compiled_pattern("passport_singapore") + assert pattern.search("K9876543") is not None + + def test_wrong_prefix_rejected(self): + pattern = get_compiled_pattern("passport_singapore") + assert pattern.search("A1234567") is None + assert pattern.search("X9876543") is None + + def test_too_few_digits_rejected(self): + pattern = get_compiled_pattern("passport_singapore") + assert pattern.search("E123456") is None # Only 6 digits + + +class TestSingaporeUEN: + """Test Singapore Unique Entity Number (UEN) detection""" + + def test_local_company_uen_8digit(self): + pattern = get_compiled_pattern("sg_uen") + # 8 digits + 1 letter (local companies) + assert pattern.search("12345678A") is not None + + def test_local_company_uen_9digit(self): + pattern = get_compiled_pattern("sg_uen") + # 9 digits + 1 letter (businesses) + assert pattern.search("123456789Z") is not None + + def test_roc_uen(self): + pattern = get_compiled_pattern("sg_uen") + # T or R + 2 digits + 2 letters + 4 digits + 1 letter + assert pattern.search("T08LL0001A") is not None + assert pattern.search("R12AB3456Z") is not None + + def test_lowercase_suffix_detected_case_insensitive(self): + pattern = get_compiled_pattern("sg_uen") + assert pattern.search("12345678a") is not None + + +class TestSingaporeBankAccount: + """Test Singapore bank account number detection""" + + def test_standard_format(self): + pattern = get_compiled_pattern("sg_bank_account") + assert pattern.search("123-45678-9") is not None + assert pattern.search("001-23456-12") is not None + assert pattern.search("999-123456-123") is not None + + def test_without_dashes_rejected(self): + pattern = get_compiled_pattern("sg_bank_account") + # Pattern requires dash format + assert pattern.search("12345678901") is None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py index f1ac6ef14b..c58584944c 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py @@ -11,8 +11,10 @@ from litellm import ModelResponse from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.noma import ( NomaGuardrail, + NomaV2Guardrail, initialize_guardrail, ) +import litellm.proxy.guardrails.guardrail_hooks.noma.noma as noma_legacy_module from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.llms.openai import AllMessageValues @@ -77,6 +79,13 @@ def mock_request_data(): class TestNomaGuardrailConfiguration: """Test configuration and initialization of Noma guardrail""" + def test_legacy_guardrail_emits_deprecation_warning(self, monkeypatch): + monkeypatch.setattr( + noma_legacy_module, "_LEGACY_NOMA_DEPRECATION_WARNED", False + ) + with pytest.warns(DeprecationWarning, match="deprecated"): + NomaGuardrail(api_key="test-api-key") + def test_init_with_config(self): """Test initializing Noma guardrail via init_guardrails_v2""" with patch.dict( @@ -167,6 +176,34 @@ class TestNomaGuardrailConfiguration: assert result.block_failures is False mock_add.assert_called_once_with(result) + def test_initialize_guardrail_use_v2_routes_to_noma_v2(self): + """Test migration routing: guardrail=noma + use_v2=True initializes NomaV2Guardrail.""" + from litellm.types.guardrails import Guardrail, LitellmParams + + litellm_params = LitellmParams( + guardrail="noma", + mode="pre_call", + use_v2=True, + api_key="test-key", + api_base="https://test.api/", + application_id="test-app", + ) + + guardrail = Guardrail( + guardrail_name="test-guardrail", + litellm_params=litellm_params, + ) + + with patch("litellm.logging_callback_manager.add_litellm_callback") as mock_add: + result = initialize_guardrail(litellm_params, guardrail) + + assert isinstance(result, NomaV2Guardrail) + assert result.api_key == "test-key" + assert result.api_base == "https://test.api" + assert result.application_id == "test-app" + mock_add.assert_called_once_with(result) + + class TestNomaApplicationIdResolution: """Tests for determining which applicationId is sent to Noma.""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma_v2.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma_v2.py new file mode 100644 index 0000000000..d5fc1bdc69 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma_v2.py @@ -0,0 +1,531 @@ +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy.guardrails.guardrail_hooks.noma import NomaV2Guardrail +from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage +from litellm.types.proxy.guardrails.guardrail_hooks.noma import ( + NomaV2GuardrailConfigModel, +) + + +@pytest.fixture +def noma_v2_guardrail(): + return NomaV2Guardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + application_id="test-app", + monitor_mode=False, + block_failures=False, + guardrail_name="test-noma-v2-guardrail", + event_hook="pre_call", + default_on=True, + ) + + +class TestNomaV2Configuration: + @pytest.mark.asyncio + async def test_provider_specific_params_include_noma_v2_fields(self): + from litellm.proxy.guardrails.guardrail_endpoints import ( + get_provider_specific_params, + ) + + provider_params = await get_provider_specific_params() + assert "noma_v2" in provider_params + + noma_v2_params = provider_params["noma_v2"] + assert noma_v2_params["ui_friendly_name"] == "Noma Security v2" + assert "api_key" in noma_v2_params + assert "api_base" in noma_v2_params + assert "application_id" in noma_v2_params + assert "monitor_mode" in noma_v2_params + assert "block_failures" in noma_v2_params + + def test_init_requires_auth_for_saas_endpoint(self): + with patch.dict(os.environ, {}, clear=True): + with pytest.raises( + ValueError, + match="requires api_key when using Noma SaaS endpoint", + ): + NomaV2Guardrail() + + def test_init_allows_missing_auth_for_self_managed_endpoint(self): + with patch.dict(os.environ, {}, clear=True): + guardrail = NomaV2Guardrail(api_base="https://self-managed.noma.local") + assert guardrail.api_key is None + + def test_init_defaults_monitor_and_block_failures(self): + with patch.dict(os.environ, {"NOMA_API_KEY": "test-api-key"}, clear=True): + guardrail = NomaV2Guardrail() + + assert guardrail.monitor_mode is False + assert guardrail.block_failures is True + + @pytest.mark.asyncio + async def test_api_key_auth_path(self, noma_v2_guardrail): + assert noma_v2_guardrail._get_authorization_header() == "Bearer test-api-key" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"action":"NONE"}' + mock_response.json.return_value = { + "action": "NONE", + } + mock_response.raise_for_status = MagicMock() + mock_post = AsyncMock(return_value=mock_response) + + with patch.object(noma_v2_guardrail.async_handler, "post", mock_post): + await noma_v2_guardrail._call_noma_scan( + payload={"inputs": {"texts": []}}, + ) + + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs["headers"]["Authorization"] == "Bearer test-api-key" + + @pytest.mark.asyncio + async def test_self_managed_path_without_api_key_omits_authorization_header(self): + guardrail = NomaV2Guardrail( + api_base="https://self-managed.noma.local", + guardrail_name="test-noma-v2-guardrail", + event_hook="pre_call", + default_on=True, + ) + assert guardrail._get_authorization_header() == "" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"action":"NONE"}' + mock_response.json.return_value = {"action": "NONE"} + mock_response.raise_for_status = MagicMock() + mock_post = AsyncMock(return_value=mock_response) + + with patch.object(guardrail.async_handler, "post", mock_post): + await guardrail._call_noma_scan(payload={"inputs": {"texts": []}}) + + sent_headers = mock_post.call_args.kwargs["headers"] + assert "Authorization" not in sent_headers + + def test_build_scan_payload_sends_raw_available_data(self, noma_v2_guardrail): + inputs = { + "texts": ["hello"], + "images": ["https://example.com/image.png"], + "structured_messages": [{"role": "user", "content": "hello"}], + "tool_calls": [{"id": "tool-1"}], + "model": "gpt-4o-mini", + } + request_data = { + "messages": [{"role": "user", "content": "hello"}], + "metadata": {"headers": {"x-noma-application-id": "header-app"}}, + "litellm_metadata": {"user_api_key_alias": "litellm-alias"}, + "litellm_call_id": "call-id-1", + } + payload = noma_v2_guardrail._build_scan_payload( + inputs=inputs, + request_data=request_data, + input_type="request", + logging_obj=None, + application_id="dynamic-app", + ) + + assert payload["inputs"] == inputs + assert payload["request_data"] == request_data + assert payload["input_type"] == "request" + assert payload["monitor_mode"] is False + assert payload["application_id"] == "dynamic-app" + assert "dynamic_params" not in payload + assert "x-noma-context" not in payload + assert "input" not in payload + + def test_build_scan_payload_deep_copies_request_data(self, noma_v2_guardrail): + request_data = { + "metadata": {"headers": {"x-noma-application-id": "header-app"}}, + "messages": [{"role": "user", "content": "hello"}], + } + payload = noma_v2_guardrail._build_scan_payload( + inputs={"texts": ["hello"]}, + request_data=request_data, + input_type="request", + logging_obj=None, + application_id="dynamic-app", + ) + + payload["request_data"]["metadata"]["headers"]["x-noma-application-id"] = "mutated-value" + payload["request_data"]["messages"][0]["content"] = "changed-content" + + assert request_data["metadata"]["headers"]["x-noma-application-id"] == "header-app" + assert request_data["messages"][0]["content"] == "hello" + + def test_build_scan_payload_passes_model_call_details_as_is(self, noma_v2_guardrail): + class _LoggingObj: + def __init__(self) -> None: + self.model_call_details = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + "call_type": "acompletion", + "litellm_call_id": "call-id-123", + "function_id": "fn-id-456", + "litellm_trace_id": "trace-id-789", + "api_key": "included-as-is", + } + + request_data = {"litellm_logging_obj": ""} + payload = noma_v2_guardrail._build_scan_payload( + inputs={"texts": ["hello"]}, + request_data=request_data, + input_type="request", + logging_obj=_LoggingObj(), + application_id="test-app", + ) + + assert payload["request_data"]["litellm_logging_obj"] == { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + "call_type": "acompletion", + "litellm_call_id": "call-id-123", + "function_id": "fn-id-456", + "litellm_trace_id": "trace-id-789", + "api_key": "included-as-is", + } + assert "logging_obj" not in payload + assert request_data["litellm_logging_obj"] == "" + + @pytest.mark.asyncio + async def test_call_noma_scan_sanitizes_response_model_dump_object(self, noma_v2_guardrail): + import json + + class _FakeModelResponse: + def model_dump(self): + return {"id": "resp-1", "content": "ok"} + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"action":"NONE"}' + mock_response.json.return_value = {"action": "NONE"} + mock_response.raise_for_status = MagicMock() + mock_post = AsyncMock(return_value=mock_response) + + payload = { + "inputs": {"texts": ["hello"]}, + "request_data": {"response": _FakeModelResponse()}, + "input_type": "response", + "application_id": "test-app", + } + + with patch.object(noma_v2_guardrail.async_handler, "post", mock_post): + await noma_v2_guardrail._call_noma_scan(payload=payload) + + sent_payload = mock_post.call_args.kwargs["json"] + json.dumps(sent_payload) + assert sent_payload["request_data"]["response"]["id"] == "resp-1" + + def test_sanitize_payload_for_transport_falls_back_to_safe_dumps(self, noma_v2_guardrail): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.json.dumps", + side_effect=TypeError("cannot serialize"), + ): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.safe_dumps", + return_value='{"fallback": true}', + ) as mock_safe_dumps: + sanitized = noma_v2_guardrail._sanitize_payload_for_transport({"inputs": {"texts": ["hello"]}}) + + mock_safe_dumps.assert_called_once() + assert sanitized == {"fallback": True} + + def test_sanitize_payload_for_transport_logs_warning_when_payload_becomes_empty(self, noma_v2_guardrail): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.safe_json_loads", + return_value={}, + ): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.verbose_proxy_logger.warning" + ) as mock_warning: + sanitized = noma_v2_guardrail._sanitize_payload_for_transport({"inputs": {"texts": ["hello"]}}) + + assert sanitized == {} + mock_warning.assert_called_once_with( + "Noma v2 guardrail: payload serialization failed, falling back to empty payload" + ) + + def test_sanitize_payload_for_transport_logs_warning_on_non_dict_output(self, noma_v2_guardrail): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.safe_json_loads", + return_value=["not-a-dict"], + ): + with patch( + "litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2.verbose_proxy_logger.warning" + ) as mock_warning: + sanitized = noma_v2_guardrail._sanitize_payload_for_transport({"inputs": {"texts": ["hello"]}}) + + assert sanitized == {} + mock_warning.assert_called_once_with( + "Noma v2 guardrail: payload sanitization produced non-dict output (type=%s), falling back to empty payload", + "list", + ) + + def test_get_config_model_returns_noma_v2_config_model(self): + assert NomaV2Guardrail.get_config_model() is NomaV2GuardrailConfigModel + + +class TestNomaV2ActionBehavior: + def test_resolve_action_from_response_raises_on_unknown_action(self, noma_v2_guardrail): + with pytest.raises(ValueError, match="missing valid action"): + noma_v2_guardrail._resolve_action_from_response({"action": "INVALID"}) + + @pytest.mark.asyncio + async def test_native_action_none(self, noma_v2_guardrail): + inputs = {"texts": ["hello"]} + with patch.object( + noma_v2_guardrail, + "_call_noma_scan", + AsyncMock( + return_value={ + "action": "NONE", + } + ), + ): + result = await noma_v2_guardrail.apply_guardrail( + inputs=inputs, + request_data={"metadata": {}}, + input_type="request", + ) + + assert result == inputs + + @pytest.mark.asyncio + async def test_native_action_guardrail_intervened_updates_supported_fields(self, noma_v2_guardrail): + inputs = { + "texts": ["Name: Jane"], + "images": ["https://old.example/image.png"], + "tools": [{"type": "function", "function": {"name": "old_tool"}}], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "old_tool", "arguments": '{"key":"value"}'}, + } + ], + } + with patch.object( + noma_v2_guardrail, + "_call_noma_scan", + AsyncMock( + return_value={ + "action": "GUARDRAIL_INTERVENED", + "texts": ["Name: *******"], + "images": ["https://new.example/image.png"], + "tools": [{"type": "function", "function": {"name": "new_tool"}}], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "new_tool", "arguments": '{"safe":"true"}'}, + } + ], + } + ), + ): + result = await noma_v2_guardrail.apply_guardrail( + inputs=inputs, + request_data={"metadata": {}}, + input_type="request", + ) + + assert result["texts"] == ["Name: *******"] + assert result["images"] == ["https://new.example/image.png"] + assert result["tools"] == [{"type": "function", "function": {"name": "new_tool"}}] + assert result["tool_calls"] == [ + { + "id": "call_1", + "type": "function", + "function": {"name": "new_tool", "arguments": '{"safe":"true"}'}, + } + ] + + @pytest.mark.asyncio + async def test_native_action_blocked(self, noma_v2_guardrail): + inputs = {"texts": ["bad"]} + with patch.object( + noma_v2_guardrail, + "_call_noma_scan", + AsyncMock( + return_value={ + "action": "BLOCKED", + "blocked_reason": "blocked by policy", + } + ), + ): + with pytest.raises(NomaBlockedMessage) as exc_info: + await noma_v2_guardrail.apply_guardrail( + inputs=inputs, + request_data={"metadata": {}}, + input_type="request", + ) + assert exc_info.value.detail["details"]["blocked_reason"] == "blocked by policy" + + @pytest.mark.asyncio + async def test_intervened_without_modifications_returns_original_inputs(self, noma_v2_guardrail): + inputs = {"texts": ["Name: Jane"]} + with patch.object( + noma_v2_guardrail, + "_call_noma_scan", + AsyncMock( + return_value={ + "action": "GUARDRAIL_INTERVENED", + } + ), + ): + result = await noma_v2_guardrail.apply_guardrail( + inputs=inputs, + request_data={"metadata": {}}, + input_type="request", + ) + assert result == inputs + + @pytest.mark.asyncio + async def test_fail_open_on_technical_scan_failure(self, noma_v2_guardrail): + inputs = {"texts": ["hello"]} + with patch.object( + noma_v2_guardrail, + "_call_noma_scan", + AsyncMock(side_effect=Exception("network error")), + ): + result = await noma_v2_guardrail.apply_guardrail( + inputs=inputs, + request_data={"metadata": {}}, + input_type="request", + ) + + assert result == inputs + + @pytest.mark.asyncio + async def test_fail_closed_on_technical_scan_failure_when_block_failures_true(self): + guardrail = NomaV2Guardrail( + api_key="test-api-key", + block_failures=True, + guardrail_name="test-noma-v2-guardrail", + event_hook="pre_call", + default_on=True, + ) + with patch.object( + guardrail, + "_call_noma_scan", + AsyncMock(side_effect=Exception("network error")), + ): + with pytest.raises(Exception, match="network error"): + await guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + @pytest.mark.asyncio + async def test_monitor_mode_ignores_block_action(self): + guardrail = NomaV2Guardrail( + api_key="test-api-key", + monitor_mode=True, + guardrail_name="test-noma-v2-guardrail", + event_hook="pre_call", + default_on=True, + ) + call_mock = AsyncMock(return_value={"action": "BLOCKED"}) + with patch.object(guardrail, "_call_noma_scan", call_mock): + result = await guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + payload = call_mock.call_args.kwargs["payload"] + assert payload["monitor_mode"] is True + assert result == {"texts": ["hello"]} + + +class TestNomaV2ApplicationIdResolution: + @pytest.mark.asyncio + async def test_apply_guardrail_uses_dynamic_application_id(self, noma_v2_guardrail): + call_mock = AsyncMock(return_value={"action": "NONE"}) + with patch.object( + noma_v2_guardrail, + "get_guardrail_dynamic_request_body_params", + return_value={"application_id": "dynamic-app"}, + ): + with patch.object(noma_v2_guardrail, "_call_noma_scan", call_mock): + await noma_v2_guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + payload = call_mock.call_args.kwargs["payload"] + assert payload["application_id"] == "dynamic-app" + + @pytest.mark.asyncio + async def test_apply_guardrail_uses_configured_application_id(self, noma_v2_guardrail): + call_mock = AsyncMock(return_value={"action": "NONE"}) + with patch.object( + noma_v2_guardrail, + "get_guardrail_dynamic_request_body_params", + return_value={}, + ): + with patch.object(noma_v2_guardrail, "_call_noma_scan", call_mock): + await noma_v2_guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + payload = call_mock.call_args.kwargs["payload"] + assert payload["application_id"] == "test-app" + + @pytest.mark.asyncio + async def test_apply_guardrail_omits_application_id_when_not_explicit(self): + guardrail_no_config = NomaV2Guardrail( + api_key="test-api-key", + application_id=None, + guardrail_name="test-noma-v2-guardrail", + event_hook="pre_call", + default_on=True, + ) + + call_mock = AsyncMock(return_value={"action": "NONE"}) + with patch.object( + guardrail_no_config, + "get_guardrail_dynamic_request_body_params", + return_value={}, + ): + with patch.object(guardrail_no_config, "_call_noma_scan", call_mock): + await guardrail_no_config.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + payload = call_mock.call_args.kwargs["payload"] + assert "application_id" not in payload + + @pytest.mark.asyncio + async def test_apply_guardrail_ignores_request_metadata_application_id(self, noma_v2_guardrail): + noma_v2_guardrail.application_id = None + call_mock = AsyncMock(return_value={"action": "NONE"}) + request_data = { + "metadata": {"headers": {"x-noma-application-id": "header-app"}}, + "litellm_metadata": {"user_api_key_alias": "alias-app"}, + } + with patch.object( + noma_v2_guardrail, + "get_guardrail_dynamic_request_body_params", + return_value={}, + ): + with patch.object(noma_v2_guardrail, "_call_noma_scan", call_mock): + await noma_v2_guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data=request_data, + input_type="request", + ) + + payload = call_mock.call_args.kwargs["payload"] + assert "application_id" not in payload diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 23432b18ca..1d70126681 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -8,18 +8,30 @@ from litellm.types.guardrails import GuardrailEventHooks, Guardrail, LitellmPara def test_get_guardrail_initializer_from_hooks(): initializers = get_guardrail_initializer_from_hooks() - print(f"initializers: {initializers}") assert "aim" in initializers def test_guardrail_class_registry(): from litellm.proxy.guardrails.guardrail_registry import guardrail_class_registry - print(f"guardrail_class_registry: {guardrail_class_registry}") assert "aim" in guardrail_class_registry assert "aporia" in guardrail_class_registry +def test_noma_registry_resolution(): + from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaGuardrail + from litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2 import NomaV2Guardrail + from litellm.proxy.guardrails.guardrail_registry import ( + guardrail_class_registry, + guardrail_initializer_registry, + ) + + assert guardrail_class_registry["noma"] is NomaGuardrail + assert guardrail_class_registry["noma_v2"] is NomaV2Guardrail + assert "noma" in guardrail_initializer_registry + assert "noma_v2" in guardrail_initializer_registry + + def test_update_in_memory_guardrail(): handler = InMemoryGuardrailHandler() handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py index 1846ffaeb6..18dcb2b0b2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py @@ -78,3 +78,213 @@ async def test_create_duplicate_access_group_fails(): assert exc_info.value.status_code == 409 assert "already exists" in str(exc_info.value.detail) +@pytest.mark.asyncio +async def test_create_access_group_with_model_ids_tags_only_specific_deployments(): + """ + Test that using model_ids only tags the specific deployments, not all + deployments sharing the same model_name. + + Fixes: https://github.com/BerriAI/litellm/issues/21544 + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={}) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=deploy_a) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + + mock_user = UserAPIKeyAuth( + user_id="test_admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + request_data = NewModelGroupRequest( + access_group="production-models", + model_ids=["deploy-A"], + ) + + with patch("litellm.proxy.proxy_server.llm_router", MagicMock()), \ + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), \ + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new_callable=AsyncMock, + ): + response = await create_model_group(data=request_data, user_api_key_dict=mock_user) + + assert response.models_updated == 1 + assert response.model_ids == ["deploy-A"] + mock_prisma.db.litellm_proxymodeltable.find_unique.assert_called_once_with( + where={"model_id": "deploy-A"} + ) + assert mock_prisma.db.litellm_proxymodeltable.update.call_count == 1 + update_call = mock_prisma.db.litellm_proxymodeltable.update.call_args + assert update_call.kwargs["where"] == {"model_id": "deploy-A"} + + +@pytest.mark.asyncio +async def test_create_access_group_with_model_names_tags_all_deployments(): + """ + Test backward compat: model_names still tags ALL deployments sharing that model_name. + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={}) + deploy_b = MagicMock(model_id="deploy-B", model_name="gpt-4o", model_info={}) + deploy_c = MagicMock(model_id="deploy-C", model_name="gpt-4o", model_info={}) + + mock_router = Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "api_key": "fake-key"}}] + ) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock( + side_effect=[[], [deploy_a, deploy_b, deploy_c]] + ) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + + mock_user = UserAPIKeyAuth( + user_id="test_admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + request_data = NewModelGroupRequest(access_group="production-models", model_names=["gpt-4o"]) + + with patch("litellm.proxy.proxy_server.llm_router", mock_router), \ + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), \ + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new_callable=AsyncMock, + ): + response = await create_model_group(data=request_data, user_api_key_dict=mock_user) + + assert response.models_updated == 3 + assert response.model_names == ["gpt-4o"] + assert mock_prisma.db.litellm_proxymodeltable.update.call_count == 3 + + +@pytest.mark.asyncio +async def test_create_access_group_model_ids_takes_priority_over_model_names(): + """ + Test that when both model_ids and model_names are provided, model_ids is used. + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={}) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=deploy_a) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + + mock_user = UserAPIKeyAuth( + user_id="test_admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + request_data = NewModelGroupRequest( + access_group="production-models", + model_names=["gpt-4o"], + model_ids=["deploy-A"], + ) + + with patch("litellm.proxy.proxy_server.llm_router", MagicMock()), \ + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), \ + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new_callable=AsyncMock, + ): + response = await create_model_group(data=request_data, user_api_key_dict=mock_user) + + assert response.models_updated == 1 + mock_prisma.db.litellm_proxymodeltable.find_unique.assert_called_once_with( + where={"model_id": "deploy-A"} + ) + + +@pytest.mark.asyncio +async def test_create_access_group_requires_model_names_or_model_ids(): + """ + Test that creating an access group without model_names or model_ids fails. + """ + from fastapi import HTTPException + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + mock_user = UserAPIKeyAuth( + user_id="test_admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + request_data = NewModelGroupRequest(access_group="production-models") + + with patch("litellm.proxy.proxy_server.llm_router", MagicMock()), \ + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await create_model_group(data=request_data, user_api_key_dict=mock_user) + assert exc_info.value.status_code == 400 + assert "model_names or model_ids" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_create_access_group_invalid_model_id_returns_400(): + """ + Test that passing a non-existent model_id returns 400 error. + """ + from fastapi import HTTPException + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None) + + mock_user = UserAPIKeyAuth( + user_id="test_admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + request_data = NewModelGroupRequest( + access_group="production-models", + model_ids=["non-existent-id"], + ) + + with patch("litellm.proxy.proxy_server.llm_router", MagicMock()), \ + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), \ + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new_callable=AsyncMock, + ): + with pytest.raises(HTTPException) as exc_info: + await create_model_group(data=request_data, user_api_key_dict=mock_user) + assert exc_info.value.status_code == 400 + assert "non-existent-id" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 7bebe00d61..977304f732 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -84,7 +84,7 @@ class TestProxyBaseLLMRequestProcessing: """ Test that hierarchical router settings are stored as router_settings_override instead of creating a full user_config with model_list. - + This approach avoids expensive per-request Router instantiation by passing settings as kwargs overrides to the main router. """ @@ -114,7 +114,7 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) - + mock_router_settings = { "routing_strategy": "least-busy", "timeout": 30.0, @@ -134,7 +134,10 @@ class TestProxyBaseLLMRequestProcessing: route_type = "acompletion" - returned_data, logging_obj = await processing_obj.common_processing_pre_call_logic( + ( + returned_data, + logging_obj, + ) = await processing_obj.common_processing_pre_call_logic( request=mock_request, general_settings=mock_general_settings, user_api_key_dict=mock_user_api_key_dict, @@ -156,7 +159,7 @@ class TestProxyBaseLLMRequestProcessing: # This allows passing them as kwargs to the main router instead of creating a new one assert "router_settings_override" in returned_data assert "user_config" not in returned_data - + router_settings_override = returned_data["router_settings_override"] assert router_settings_override["routing_strategy"] == "least-busy" assert router_settings_override["timeout"] == 30.0 @@ -173,34 +176,39 @@ class TestProxyBaseLLMRequestProcessing: # Test with stream timeout header headers_with_timeout = {"x-litellm-stream-timeout": "30.5"} - result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_timeout) + result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request( + headers_with_timeout + ) assert result == 30.5 - + # Test without stream timeout header headers_without_timeout = {} - result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_without_timeout) + result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request( + headers_without_timeout + ) assert result is None - + # Test with invalid header value (should raise ValueError when converting to float) headers_with_invalid = {"x-litellm-stream-timeout": "invalid"} with pytest.raises(ValueError): - LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid) + LiteLLMProxyRequestSetup._get_stream_timeout_from_request( + headers_with_invalid + ) @pytest.mark.asyncio async def test_add_litellm_data_to_request_with_stream_timeout_header(self): """ - Test that x-litellm-stream-timeout header gets processed and added to request data + Test that x-litellm-stream-timeout header gets processed and added to request data when calling add_litellm_data_to_request. """ - from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request # Create test data with a basic completion request test_data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } - + # Mock request with stream timeout header mock_request = MagicMock(spec=Request) mock_request.headers = {"x-litellm-stream-timeout": "45.0"} @@ -208,7 +216,7 @@ class TestProxyBaseLLMRequestProcessing: mock_request.method = "POST" mock_request.query_params = {} mock_request.client = None - + # Create a minimal mock with just the required attributes mock_user_api_key_dict = MagicMock() mock_user_api_key_dict.api_key = "test_api_key_hash" @@ -232,10 +240,10 @@ class TestProxyBaseLLMRequestProcessing: mock_user_api_key_dict.model_max_budget = None mock_user_api_key_dict.parent_otel_span = None mock_user_api_key_dict.team_model_aliases = None - + general_settings = {} mock_proxy_config = MagicMock() - + # Call the actual function that processes headers and adds data result_data = await add_litellm_data_to_request( data=test_data, @@ -245,11 +253,11 @@ class TestProxyBaseLLMRequestProcessing: version=None, proxy_config=mock_proxy_config, ) - + # Verify that stream_timeout was extracted from header and added to request data assert "stream_timeout" in result_data assert result_data["stream_timeout"] == 45.0 - + # Verify that the original test data is preserved assert result_data["model"] == "gpt-3.5-turbo" assert result_data["messages"] == [{"role": "user", "content": "Hello"}] @@ -269,7 +277,7 @@ class TestProxyBaseLLMRequestProcessing: mock_user_api_key_dict.rpm_limit = None mock_user_api_key_dict.max_budget = None mock_user_api_key_dict.spend = 0 - + # Create logging object with cost breakdown including discount logging_obj = LiteLLMLoggingObj( model="vertex_ai/gemini-pro", @@ -280,7 +288,7 @@ class TestProxyBaseLLMRequestProcessing: litellm_call_id="test-call-id", function_id="test-function-id", ) - + # Set cost breakdown with discount information logging_obj.set_cost_breakdown( input_cost=0.00005, @@ -291,7 +299,7 @@ class TestProxyBaseLLMRequestProcessing: discount_percent=0.05, discount_amount=0.000005, ) - + # Call get_custom_headers with discount info headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=mock_user_api_key_dict, @@ -299,14 +307,14 @@ class TestProxyBaseLLMRequestProcessing: response_cost=0.000095, litellm_logging_obj=logging_obj, ) - + # Verify discount headers are present assert "x-litellm-response-cost" in headers assert float(headers["x-litellm-response-cost"]) == 0.000095 - + assert "x-litellm-response-cost-original" in headers assert float(headers["x-litellm-response-cost-original"]) == 0.0001 - + assert "x-litellm-response-cost-discount-amount" in headers assert float(headers["x-litellm-response-cost-discount-amount"]) == 0.000005 @@ -324,7 +332,7 @@ class TestProxyBaseLLMRequestProcessing: mock_user_api_key_dict.rpm_limit = None mock_user_api_key_dict.max_budget = None mock_user_api_key_dict.spend = 0 - + # Create logging object without discount logging_obj = LiteLLMLoggingObj( model="gpt-3.5-turbo", @@ -335,7 +343,7 @@ class TestProxyBaseLLMRequestProcessing: litellm_call_id="test-call-id", function_id="test-function-id", ) - + # Set cost breakdown without discount information logging_obj.set_cost_breakdown( input_cost=0.00005, @@ -343,7 +351,7 @@ class TestProxyBaseLLMRequestProcessing: total_cost=0.0001, cost_for_built_in_tools_cost_usd_dollar=0.0, ) - + # Call get_custom_headers headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=mock_user_api_key_dict, @@ -351,11 +359,11 @@ class TestProxyBaseLLMRequestProcessing: response_cost=0.0001, litellm_logging_obj=logging_obj, ) - + # Verify discount headers are NOT present assert "x-litellm-response-cost" in headers assert float(headers["x-litellm-response-cost"]) == 0.0001 - + # Discount headers should not be in the final dict assert "x-litellm-response-cost-original" not in headers assert "x-litellm-response-cost-discount-amount" not in headers @@ -374,7 +382,7 @@ class TestProxyBaseLLMRequestProcessing: mock_user_api_key_dict.rpm_limit = None mock_user_api_key_dict.max_budget = None mock_user_api_key_dict.spend = 0 - + # Create logging object with margin logging_obj = LiteLLMLoggingObj( model="gpt-4", @@ -394,20 +402,20 @@ class TestProxyBaseLLMRequestProcessing: margin_percent=0.10, margin_total_amount=0.00001, ) - + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=mock_user_api_key_dict, response_cost=0.00011, litellm_logging_obj=logging_obj, ) - + # Verify margin headers are present assert "x-litellm-response-cost" in headers assert float(headers["x-litellm-response-cost"]) == 0.00011 - + assert "x-litellm-response-cost-margin-amount" in headers assert float(headers["x-litellm-response-cost-margin-amount"]) == 0.00001 - + assert "x-litellm-response-cost-margin-percent" in headers assert float(headers["x-litellm-response-cost-margin-percent"]) == 0.10 @@ -425,7 +433,7 @@ class TestProxyBaseLLMRequestProcessing: mock_user_api_key_dict.rpm_limit = None mock_user_api_key_dict.max_budget = None mock_user_api_key_dict.spend = 0 - + # Create logging object without margin logging_obj = LiteLLMLoggingObj( model="gpt-4", @@ -442,13 +450,13 @@ class TestProxyBaseLLMRequestProcessing: total_cost=0.0001, cost_for_built_in_tools_cost_usd_dollar=0.0, ) - + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=mock_user_api_key_dict, response_cost=0.0001, litellm_logging_obj=logging_obj, ) - + # Verify margin headers are not present assert "x-litellm-response-cost-margin-amount" not in headers assert "x-litellm-response-cost-margin-percent" not in headers @@ -480,13 +488,18 @@ class TestProxyBaseLLMRequestProcessing: discount_percent=0.05, discount_amount=0.000005, ) - - original_cost, discount_amount, margin_total_amount, margin_percent = _get_cost_breakdown_from_logging_obj(logging_obj) + + ( + original_cost, + discount_amount, + margin_total_amount, + margin_percent, + ) = _get_cost_breakdown_from_logging_obj(logging_obj) assert original_cost == 0.0001 assert discount_amount == 0.000005 assert margin_total_amount is None assert margin_percent is None - + # Test with margin info logging_obj_with_margin = LiteLLMLoggingObj( model="gpt-4", @@ -506,13 +519,18 @@ class TestProxyBaseLLMRequestProcessing: margin_percent=0.10, margin_total_amount=0.00001, ) - - original_cost, discount_amount, margin_total_amount, margin_percent = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin) + + ( + original_cost, + discount_amount, + margin_total_amount, + margin_percent, + ) = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin) assert original_cost == 0.0001 assert discount_amount is None assert margin_total_amount == 0.00001 assert margin_percent == 0.10 - + # Test with no discount or margin info logging_obj_no_discount = LiteLLMLoggingObj( model="gpt-3.5-turbo", @@ -529,15 +547,25 @@ class TestProxyBaseLLMRequestProcessing: total_cost=0.0001, cost_for_built_in_tools_cost_usd_dollar=0.0, ) - - original_cost, discount_amount, margin_total_amount, margin_percent = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount) + + ( + original_cost, + discount_amount, + margin_total_amount, + margin_percent, + ) = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount) assert original_cost is None assert discount_amount is None assert margin_total_amount is None assert margin_percent is None - + # Test with None logging object - original_cost, discount_amount, margin_total_amount, margin_percent = _get_cost_breakdown_from_logging_obj(None) + ( + original_cost, + discount_amount, + margin_total_amount, + margin_percent, + ) = _get_cost_breakdown_from_logging_obj(None) assert original_cost is None assert discount_amount is None assert margin_total_amount is None @@ -546,7 +574,7 @@ class TestProxyBaseLLMRequestProcessing: def test_get_custom_headers_key_spend_includes_response_cost(self): """ Test that x-litellm-key-spend header includes the current request's response_cost. - + This ensures that the spend header reflects the updated spend including the current request, even though spend tracking updates happen asynchronously after the response. """ @@ -564,10 +592,12 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-1", response_cost=response_cost_1, ) - + assert "x-litellm-key-spend" in headers_1 expected_spend_1 = 0.001 + 0.0005 # Initial spend + current request cost - assert float(headers_1["x-litellm-key-spend"]) == pytest.approx(expected_spend_1, abs=1e-10) + assert float(headers_1["x-litellm-key-spend"]) == pytest.approx( + expected_spend_1, abs=1e-10 + ) assert float(headers_1["x-litellm-response-cost"]) == response_cost_1 # Test case 2: response_cost is provided as string @@ -577,10 +607,12 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-2", response_cost=response_cost_2, ) - + assert "x-litellm-key-spend" in headers_2 expected_spend_2 = 0.001 + 0.0003 # Initial spend + current request cost - assert float(headers_2["x-litellm-key-spend"]) == pytest.approx(expected_spend_2, abs=1e-10) + assert float(headers_2["x-litellm-key-spend"]) == pytest.approx( + expected_spend_2, abs=1e-10 + ) # Test case 3: response_cost is None (should use original spend) headers_3 = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -588,9 +620,11 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-3", response_cost=None, ) - + assert "x-litellm-key-spend" in headers_3 - assert float(headers_3["x-litellm-key-spend"]) == 0.001 # Should use original spend + assert ( + float(headers_3["x-litellm-key-spend"]) == 0.001 + ) # Should use original spend # Test case 4: response_cost is 0 (should not change spend) headers_4 = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -598,9 +632,11 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-4", response_cost=0.0, ) - + assert "x-litellm-key-spend" in headers_4 - assert float(headers_4["x-litellm-key-spend"]) == 0.001 # Should remain unchanged for 0 cost + assert ( + float(headers_4["x-litellm-key-spend"]) == 0.001 + ) # Should remain unchanged for 0 cost # Test case 5: user_api_key_dict.spend is None (should default to 0.0) mock_user_api_key_dict.spend = None @@ -609,7 +645,7 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-5", response_cost=0.0002, ) - + assert "x-litellm-key-spend" in headers_5 assert float(headers_5["x-litellm-key-spend"]) == 0.0002 # 0.0 + 0.0002 @@ -620,9 +656,11 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-6", response_cost=-0.0001, # Negative cost (should not be added) ) - + assert "x-litellm-key-spend" in headers_6 - assert float(headers_6["x-litellm-key-spend"]) == 0.001 # Should use original spend + assert ( + float(headers_6["x-litellm-key-spend"]) == 0.001 + ) # Should use original spend # Test case 7: response_cost is invalid string (should fallback to original spend) headers_7 = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -630,9 +668,77 @@ class TestProxyBaseLLMRequestProcessing: call_id="test-call-id-7", response_cost="invalid", # Invalid string ) - + assert "x-litellm-key-spend" in headers_7 - assert float(headers_7["x-litellm-key-spend"]) == 0.001 # Should use original spend on error + assert ( + float(headers_7["x-litellm-key-spend"]) == 0.001 + ) # Should use original spend on error + + @pytest.mark.asyncio + async def test_queue_time_seconds_is_set_in_metadata(self, monkeypatch): + """ + Test that queue_time_seconds is correctly calculated and stored in metadata + after add_litellm_data_to_request populates arrival_time. + + This verifies the fix for the bug where queue_time_seconds was always None + because arrival_time was read BEFORE add_litellm_data_to_request set it. + """ + processing_obj = ProxyBaseLLMRequestProcessing(data={}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + mock_request.url = MagicMock() + mock_request.url.path = "/v1/chat/completions" + + async def mock_add_litellm_data_to_request(*args, **kwargs): + data = kwargs.get("data", args[0] if args else {}) + # Simulate what add_litellm_data_to_request does: set arrival_time + import time + + data["proxy_server_request"] = { + "url": "/v1/chat/completions", + "method": "POST", + "headers": {}, + "body": {}, + "arrival_time": time.time() - 0.5, # Simulate request arrived 0.5s ago + } + data["metadata"] = data.get("metadata", {}) + return data + + async def mock_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + mock_general_settings = {} + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_proxy_config = MagicMock(spec=ProxyConfig) + route_type = "acompletion" + + ( + returned_data, + logging_obj, + ) = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings=mock_general_settings, + user_api_key_dict=mock_user_api_key_dict, + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type=route_type, + ) + + # Verify queue_time_seconds is set and non-negative + metadata = returned_data.get("metadata", {}) + assert ( + "queue_time_seconds" in metadata + ), "queue_time_seconds should be set in metadata" + assert ( + metadata["queue_time_seconds"] >= 0.5 + ), f"queue_time_seconds should be at least 0.5, got {metadata['queue_time_seconds']}" @pytest.mark.asyncio @@ -695,19 +801,19 @@ class TestCommonRequestProcessingHelpers: Test that when the first chunk is an error, a JSON error response is returned instead of an SSE streaming response """ + async def mock_generator(): yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' yield 'data: {"content": "more data"}\n\n' yield "data: [DONE]\n\n" - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) # Should return JSONResponse instead of StreamingResponse assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_403_FORBIDDEN # Verify the response is in standard JSON error format import json + body = json.loads(response.body.decode()) assert "error" in body assert body["error"]["code"] == 403 @@ -719,9 +825,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "second part"}\n\n' yield "data: [DONE]\n\n" - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK content = await self.consume_stream(response) assert content == [ @@ -736,9 +840,7 @@ class TestCommonRequestProcessingHelpers: yield # Implicitly raises StopAsyncIteration - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK content = await self.consume_stream(response) assert content == [] @@ -780,17 +882,17 @@ class TestCommonRequestProcessingHelpers: """ Test that when the first chunk contains a string error code, a JSON error response is returned """ + async def mock_generator(): yield 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n' yield "data: [DONE]\n\n" - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS # Verify the response is in standard JSON error format import json + body = json.loads(response.body.decode()) assert "error" in body assert body["error"]["code"] == "429" @@ -829,9 +931,7 @@ class TestCommonRequestProcessingHelpers: async def mock_generator(): yield "data: [DONE]\n\n" - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK # Default status content = await self.consume_stream(response) assert content == ["data: [DONE]\n\n"] @@ -842,9 +942,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "actual data"}\n\n' yield "data: [DONE]\n\n" - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK # Default status content = await self.consume_stream(response) assert content == [ @@ -855,7 +953,6 @@ class TestCommonRequestProcessingHelpers: async def test_create_streaming_response_all_chunks_have_dd_trace(self): """Test that all stream chunks are wrapped with dd trace at the streaming generator level""" - import json from unittest.mock import patch # Create a mock tracer @@ -873,9 +970,7 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == 200 @@ -930,9 +1025,7 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_response( - mock_generator(), "text/event-stream", {} - ) + response = await create_response(mock_generator(), "text/event-stream", {}) # Should return JSONResponse instead of StreamingResponse assert isinstance(response, JSONResponse) @@ -940,6 +1033,7 @@ class TestCommonRequestProcessingHelpers: # Verify the response is in standard JSON error format import json + body = json.loads(response.body.decode()) assert "error" in body assert body["error"]["code"] == 400 @@ -1000,7 +1094,7 @@ class TestExtractErrorFromSSEChunk: def test_extract_error_from_sse_chunk_with_invalid_json(self): """Test invalid JSON should return default error""" - chunk = 'data: {invalid json}\n\n' + chunk = "data: {invalid json}\n\n" error = _extract_error_from_sse_chunk(chunk) assert error["message"] == "Unknown error" @@ -1037,35 +1131,35 @@ class TestExtractErrorFromSSEChunk: class TestOverrideOpenAIResponseModel: """Tests for _override_openai_response_model function""" - def test_override_model_preserves_fallback_model_when_fallback_occurred_object(self): + def test_override_model_preserves_fallback_model_when_fallback_occurred_object( + self, + ): """ Test that when a fallback occurred (x-litellm-attempted-fallbacks > 0), the actual model used (fallback model) is preserved instead of being overridden with the requested model. - + This is the regression test to ensure the model being called is properly displayed when a fallback happens. """ requested_model = "gpt-4" fallback_model = "gpt-3.5-turbo" - + # Create a mock object response with fallback model # _hidden_params is an attribute (not a dict key) accessed via getattr response_obj = MagicMock() response_obj.model = fallback_model response_obj._hidden_params = { - "additional_headers": { - "x-litellm-attempted-fallbacks": 1 - } + "additional_headers": {"x-litellm-attempted-fallbacks": 1} } - + # Call the function - should preserve fallback model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model was NOT overridden - should still be the fallback model assert response_obj.model == fallback_model assert response_obj.model != requested_model @@ -1077,7 +1171,7 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" fallback_model = "claude-haiku-4-5-20251001" - + # Create a mock object response with fallback model response_obj = MagicMock() response_obj.model = fallback_model @@ -1086,14 +1180,14 @@ class TestOverrideOpenAIResponseModel: "x-litellm-attempted-fallbacks": 2 # Multiple fallbacks } } - + # Call the function - should preserve fallback model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model was NOT overridden - should still be the fallback model assert response_obj.model == fallback_model assert response_obj.model != requested_model @@ -1105,19 +1199,19 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" downstream_model = "gpt-3.5-turbo" - + # Create a dict response without fallback # For dict responses, _hidden_params won't be found via getattr, # so the fallback check won't trigger and model will be overridden response_obj = {"model": downstream_model} - + # Call the function - should override to requested model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model WAS overridden to requested model assert response_obj["model"] == requested_model @@ -1128,21 +1222,21 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" downstream_model = "gpt-3.5-turbo" - + # Create a mock object response without fallback response_obj = MagicMock() response_obj.model = downstream_model response_obj._hidden_params = { "additional_headers": {} # No attempted_fallbacks header } - + # Call the function - should override to requested model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model WAS overridden to requested model assert response_obj.model == requested_model @@ -1153,7 +1247,7 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" downstream_model = "gpt-3.5-turbo" - + # Create a mock object response response_obj = MagicMock() response_obj.model = downstream_model @@ -1162,14 +1256,14 @@ class TestOverrideOpenAIResponseModel: "x-litellm-attempted-fallbacks": 0 # Zero means no fallback occurred } } - + # Call the function - should override to requested model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model WAS overridden to requested model assert response_obj.model == requested_model @@ -1180,23 +1274,21 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" downstream_model = "gpt-3.5-turbo" - + # Create a mock object response response_obj = MagicMock() response_obj.model = downstream_model response_obj._hidden_params = { - "additional_headers": { - "x-litellm-attempted-fallbacks": None - } + "additional_headers": {"x-litellm-attempted-fallbacks": None} } - + # Call the function - should override to requested model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model WAS overridden to requested model assert response_obj.model == requested_model @@ -1207,19 +1299,19 @@ class TestOverrideOpenAIResponseModel: """ requested_model = "gpt-4" downstream_model = "gpt-3.5-turbo" - + # Create a mock object response without _hidden_params response_obj = MagicMock() response_obj.model = downstream_model # Don't set _hidden_params - getattr will return {} - + # Call the function - should override to requested model _override_openai_response_model( response_obj=response_obj, requested_model=requested_model, log_context="test_context", ) - + # Verify the model WAS overridden to requested model assert response_obj.model == requested_model @@ -1229,34 +1321,30 @@ class TestOverrideOpenAIResponseModel: without modifying the response. """ fallback_model = "gpt-3.5-turbo" - + # Create a mock object response response_obj = MagicMock() response_obj.model = fallback_model response_obj._hidden_params = { - "additional_headers": { - "x-litellm-attempted-fallbacks": 1 - } + "additional_headers": {"x-litellm-attempted-fallbacks": 1} } - + # Call the function with None requested_model _override_openai_response_model( response_obj=response_obj, requested_model=None, log_context="test_context", ) - + # Verify the model was not changed assert response_obj.model == fallback_model - + # Call with empty string _override_openai_response_model( response_obj=response_obj, requested_model="", log_context="test_context", ) - + # Verify the model was not changed assert response_obj.model == fallback_model - - diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index d5d20ce0b0..31baab0092 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -663,6 +663,119 @@ class TestProxySettingEndpoints: assert "UI_LOGO_PATH" in updated_config["environment_variables"] assert mock_proxy_config["save_call_count"]() == 1 + def test_update_ui_theme_settings_with_favicon( + self, mock_proxy_config, mock_auth, monkeypatch + ): + """Test updating UI theme settings with favicon_url""" + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.store_model_in_db", True + ) + + new_theme = { + "logo_url": "https://example.com/new-logo.png", + "favicon_url": "https://example.com/custom-favicon.ico", + } + + response = client.patch( + "/update/ui_theme_settings", json=new_theme + ) + + assert response.status_code == 200 + data = response.json() + + assert data["status"] == "success" + assert ( + data["theme_config"]["logo_url"] + == "https://example.com/new-logo.png" + ) + assert ( + data["theme_config"]["favicon_url"] + == "https://example.com/custom-favicon.ico" + ) + + updated_config = mock_proxy_config["config"] + assert "UI_LOGO_PATH" in updated_config["environment_variables"] + assert ( + "LITELLM_FAVICON_URL" + in updated_config["environment_variables"] + ) + assert ( + updated_config["environment_variables"][ + "LITELLM_FAVICON_URL" + ] + == "https://example.com/custom-favicon.ico" + ) + + def test_update_ui_theme_settings_clear_favicon( + self, mock_proxy_config, mock_auth, monkeypatch + ): + """Test clearing favicon_url from UI theme settings""" + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.store_model_in_db", True + ) + + new_theme = { + "favicon_url": "https://example.com/custom-favicon.ico", + } + response = client.patch( + "/update/ui_theme_settings", json=new_theme + ) + assert response.status_code == 200 + + clear_theme = {"favicon_url": None} + response = client.patch( + "/update/ui_theme_settings", json=clear_theme + ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "LITELLM_FAVICON_URL" not in os.environ + + def test_get_ui_theme_settings_includes_favicon_schema( + self, mock_proxy_config + ): + """Test UI theme settings includes favicon_url in schema""" + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + data = response.json() + + assert "values" in data + assert "field_schema" in data + assert "properties" in data["field_schema"] + assert "favicon_url" in data["field_schema"]["properties"] + assert ( + "description" + in data["field_schema"]["properties"]["favicon_url"] + ) + + def test_get_ui_theme_settings_with_favicon_configured( + self, mock_proxy_config + ): + """Test getting UI theme settings when favicon is configured""" + mock_proxy_config["config"]["litellm_settings"][ + "ui_theme_config" + ] = { + "logo_url": "https://example.com/logo.png", + "favicon_url": "https://example.com/favicon.ico", + } + + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + data = response.json() + + assert ( + data["values"]["logo_url"] + == "https://example.com/logo.png" + ) + assert ( + data["values"]["favicon_url"] + == "https://example.com/favicon.ico" + ) + def test_get_ui_settings(self, mock_auth, monkeypatch): """Test retrieving UI settings with allowlist sanitization""" from unittest.mock import AsyncMock, MagicMock diff --git a/tests/test_litellm/test_get_blog_posts.py b/tests/test_litellm/test_get_blog_posts.py new file mode 100644 index 0000000000..a17d78e0bb --- /dev/null +++ b/tests/test_litellm/test_get_blog_posts.py @@ -0,0 +1,165 @@ +"""Tests for GetBlogPosts utility class.""" +import json +import time +from unittest.mock import MagicMock, patch + +import pytest + +import litellm +from litellm.litellm_core_utils.get_blog_posts import ( + BlogPost, + BlogPostsResponse, + GetBlogPosts, + get_blog_posts, +) + +SAMPLE_RESPONSE = { + "posts": [ + { + "title": "Test Post", + "description": "A test post.", + "date": "2026-01-01", + "url": "https://www.litellm.ai/blog/test", + } + ] +} + + +@pytest.fixture(autouse=True) +def reset_blog_posts_cache(): + GetBlogPosts._cached_posts = None + GetBlogPosts._last_fetch_time = 0.0 + yield + GetBlogPosts._cached_posts = None + GetBlogPosts._last_fetch_time = 0.0 + + +def test_load_local_blog_posts_returns_list(): + posts = GetBlogPosts.load_local_blog_posts() + assert isinstance(posts, list) + assert len(posts) > 0 + first = posts[0] + assert "title" in first + assert "description" in first + assert "date" in first + assert "url" in first + + +def test_validate_blog_posts_valid(): + assert GetBlogPosts.validate_blog_posts(SAMPLE_RESPONSE) is True + + +def test_validate_blog_posts_missing_posts_key(): + assert GetBlogPosts.validate_blog_posts({"other": []}) is False + + +def test_validate_blog_posts_empty_list(): + assert GetBlogPosts.validate_blog_posts({"posts": []}) is False + + +def test_validate_blog_posts_not_dict(): + assert GetBlogPosts.validate_blog_posts("not a dict") is False + + +def test_get_blog_posts_success(): + """Fetches from remote on first call.""" + mock_response = MagicMock() + mock_response.json.return_value = SAMPLE_RESPONSE + mock_response.raise_for_status = MagicMock() + + with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", return_value=mock_response): + posts = get_blog_posts(url=litellm.blog_posts_url) + + assert len(posts) == 1 + assert posts[0]["title"] == "Test Post" + + +def test_get_blog_posts_network_error_falls_back_to_local(): + """Falls back to local backup on network error.""" + with patch( + "litellm.litellm_core_utils.get_blog_posts.httpx.get", + side_effect=Exception("Network error"), + ): + posts = get_blog_posts(url=litellm.blog_posts_url) + + assert isinstance(posts, list) + assert len(posts) > 0 + + +def test_get_blog_posts_invalid_json_falls_back_to_local(): + """Falls back when remote returns non-dict.""" + mock_response = MagicMock() + mock_response.json.return_value = "not a dict" + mock_response.raise_for_status = MagicMock() + + with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", return_value=mock_response): + posts = get_blog_posts(url=litellm.blog_posts_url) + + assert isinstance(posts, list) + assert len(posts) > 0 + + +def test_get_blog_posts_ttl_cache_not_refetched(): + """Within TTL window, does not re-fetch.""" + GetBlogPosts._cached_posts = SAMPLE_RESPONSE["posts"] + GetBlogPosts._last_fetch_time = time.time() # just now + + call_count = 0 + + def mock_get(*args, **kwargs): + nonlocal call_count + call_count += 1 + m = MagicMock() + m.json.return_value = SAMPLE_RESPONSE + m.raise_for_status = MagicMock() + return m + + with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", side_effect=mock_get): + posts = get_blog_posts(url=litellm.blog_posts_url) + + assert call_count == 0 # cache hit, no fetch + assert len(posts) == 1 + + +def test_get_blog_posts_ttl_expired_refetches(): + """After TTL window, re-fetches from remote.""" + GetBlogPosts._cached_posts = SAMPLE_RESPONSE["posts"] + GetBlogPosts._last_fetch_time = time.time() - 7200 # 2 hours ago + + mock_response = MagicMock() + mock_response.json.return_value = SAMPLE_RESPONSE + mock_response.raise_for_status = MagicMock() + + with patch( + "litellm.litellm_core_utils.get_blog_posts.httpx.get", return_value=mock_response + ) as mock_get: + posts = get_blog_posts(url=litellm.blog_posts_url) + + mock_get.assert_called_once() + assert len(posts) == 1 + + +def test_get_blog_posts_local_env_var_skips_remote(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_BLOG_POSTS", "true") + with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get") as mock_get: + posts = get_blog_posts(url=litellm.blog_posts_url) + mock_get.assert_not_called() + assert isinstance(posts, list) + assert len(posts) > 0 + + +def test_blog_post_pydantic_model(): + post = BlogPost( + title="T", + description="D", + date="2026-01-01", + url="https://example.com", + ) + assert post.title == "T" + + +def test_blog_posts_response_pydantic_model(): + resp = BlogPostsResponse( + posts=[BlogPost(title="T", description="D", date="2026-01-01", url="https://x.com")] + ) + assert len(resp.posts) == 1 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 3ae4588278..35cb290fcc 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -13,12 +13,12 @@ sys.path.insert( import litellm from litellm.proxy.utils import is_valid_api_key from litellm.types.utils import ( + CallTypes, Delta, LlmProviders, ModelResponseStream, StreamingChoices, ) -from litellm.types.utils import CallTypes from litellm.utils import ( ProviderConfigManager, TextCompletionStreamWrapper, @@ -606,10 +606,14 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_above_200k_tokens": {"type": "number"}, "cache_read_input_token_cost_flex": {"type": "number"}, "cache_read_input_token_cost_priority": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_token_flex": {"type": "number"}, "input_cost_per_token_priority": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, + "input_cost_per_audio_token_priority": {"type": "number"}, "output_cost_per_token_flex": {"type": "number"}, "output_cost_per_token_priority": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_pixel": {"type": "number"}, "input_cost_per_query": {"type": "number"}, "input_cost_per_request": {"type": "number"}, @@ -644,6 +648,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "max_video_length": {"type": "number"}, "max_videos_per_prompt": {"type": "number"}, "metadata": {"type": "object"}, + "provider_specific_entry": {"type": "object"}, "mode": { "type": "string", "enum": [ @@ -714,6 +719,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_preset": {"type": "boolean"}, "tool_use_system_prompt_tokens": {"type": "number"}, "tpm": {"type": "number"}, + "provider_specific_entry": {"type": "object"}, "supported_endpoints": { "type": "array", "items": { @@ -802,8 +808,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, } - prod_json = "./model_prices_and_context_window.json" - # prod_json = "../../model_prices_and_context_window.json" + prod_json = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) assert isinstance(actual_json, dict) @@ -2337,7 +2342,7 @@ def test_register_model_with_scientific_notation(): Test that the register_model function can handle scientific notation in the model name. """ import uuid - + # Use a truly unique model name with uuid to avoid conflicts when tests run in parallel test_model_name = f"test-scientific-notation-model-{uuid.uuid4().hex[:12]}" @@ -2981,8 +2986,8 @@ class TestProxyLoggingBudgetAlerts: via metadata.soft_budget_alerting_emails to work even when global alerting is disabled. """ from litellm.caching.caching import DualCache - from litellm.proxy.utils import ProxyLogging from litellm.proxy._types import CallInfo, Litellm_EntityType + from litellm.proxy.utils import ProxyLogging proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) proxy_logging.alerting = None # Global alerting is disabled @@ -3018,8 +3023,8 @@ class TestProxyLoggingBudgetAlerts: and do not send emails when alerting is None. """ from litellm.caching.caching import DualCache - from litellm.proxy.utils import ProxyLogging from litellm.proxy._types import CallInfo, Litellm_EntityType + from litellm.proxy.utils import ProxyLogging proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) proxy_logging.alerting = None @@ -3050,8 +3055,8 @@ class TestProxyLoggingBudgetAlerts: Test that soft_budget alerts with empty alert_emails list still respect alerting=None. """ from litellm.caching.caching import DualCache - from litellm.proxy.utils import ProxyLogging from litellm.proxy._types import CallInfo, Litellm_EntityType + from litellm.proxy.utils import ProxyLogging proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) proxy_logging.alerting = None @@ -3554,3 +3559,43 @@ class TestMetadataNoneHandling: litellm_params = {"metadata": None} metadata = litellm_params.get("metadata") or {} assert metadata == {} + + +class TestValidateAndFixThinkingParam: + """Tests for validate_and_fix_thinking_param.""" + + def test_none_returns_none(self): + from litellm.utils import validate_and_fix_thinking_param + + assert validate_and_fix_thinking_param(thinking=None) is None + + def test_already_snake_case(self): + from litellm.utils import validate_and_fix_thinking_param + + thinking = {"type": "enabled", "budget_tokens": 32000} + result = validate_and_fix_thinking_param(thinking=thinking) + assert result == {"type": "enabled", "budget_tokens": 32000} + + def test_camel_case_normalized(self): + from litellm.utils import validate_and_fix_thinking_param + + thinking = {"type": "enabled", "budgetTokens": 32000} + result = validate_and_fix_thinking_param(thinking=thinking) + assert result == {"type": "enabled", "budget_tokens": 32000} + assert "budgetTokens" not in result + + def test_both_keys_snake_case_wins(self): + from litellm.utils import validate_and_fix_thinking_param + + thinking = {"type": "enabled", "budget_tokens": 10000, "budgetTokens": 50000} + result = validate_and_fix_thinking_param(thinking=thinking) + assert result == {"type": "enabled", "budget_tokens": 10000} + assert "budgetTokens" not in result + + def test_original_dict_not_mutated(self): + from litellm.utils import validate_and_fix_thinking_param + + thinking = {"type": "enabled", "budgetTokens": 32000} + validate_and_fix_thinking_param(thinking=thinking) + assert "budgetTokens" in thinking + assert "budget_tokens" not in thinking diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 5545a138dd..661cdd8709 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -14,6 +14,7 @@ import litellm from litellm.cost_calculator import default_video_cost_calculator from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.gemini.videos.transformation import GeminiVideoConfig from litellm.llms.openai.videos.transformation import OpenAIVideoConfig @@ -872,6 +873,83 @@ def test_openai_transform_video_content_request_empty_params(): assert params == {} +@pytest.mark.parametrize( + "variant,expected_suffix", + [ + ("thumbnail", "?variant=thumbnail"), + ("spritesheet", "?variant=spritesheet"), + ], +) +def test_openai_transform_video_content_request_with_variant(variant, expected_suffix): + """OpenAI content transform should append ?variant= when variant is provided.""" + config = OpenAIVideoConfig() + url, params = config.transform_video_content_request( + video_id="video_123", + api_base="https://api.openai.com/v1/videos", + litellm_params={}, + headers={}, + variant=variant, + ) + + assert url == f"https://api.openai.com/v1/videos/video_123/content{expected_suffix}" + assert params == {} + + +def test_openai_transform_video_content_request_variant_none_no_query_param(): + """OpenAI content transform should NOT append ?variant= when variant is None.""" + config = OpenAIVideoConfig() + url, params = config.transform_video_content_request( + video_id="video_123", + api_base="https://api.openai.com/v1/videos", + litellm_params={}, + headers={}, + variant=None, + ) + + assert "variant" not in url + assert url == "https://api.openai.com/v1/videos/video_123/content" + + +def test_video_content_handler_passes_variant_to_url(): + """HTTP handler should pass variant through to the final URL.""" + from litellm.llms.custom_httpx.http_handler import HTTPHandler + from litellm.types.router import GenericLiteLLMParams + + if hasattr(litellm, "in_memory_llm_clients_cache"): + litellm.in_memory_llm_clients_cache.flush_cache() + + handler = BaseLLMHTTPHandler() + config = OpenAIVideoConfig() + + mock_client = MagicMock(spec=HTTPHandler) + mock_response = MagicMock() + mock_response.content = b"thumbnail-bytes" + mock_client.get.return_value = mock_response + + with patch( + "litellm.llms.custom_httpx.llm_http_handler._get_httpx_client", + return_value=mock_client, + ): + result = handler.video_content_handler( + video_id="video_abc", + video_content_provider_config=config, + custom_llm_provider="openai", + litellm_params=GenericLiteLLMParams( + api_base="https://api.openai.com/v1" + ), + logging_obj=MagicMock(), + timeout=5.0, + api_key="sk-test", + client=mock_client, + _is_async=False, + variant="thumbnail", + ) + + assert result == b"thumbnail-bytes" + called_url = mock_client.get.call_args.kwargs["url"] + assert called_url == "https://api.openai.com/v1/videos/video_abc/content?variant=thumbnail" + + def test_video_content_handler_uses_get_for_openai(): """HTTP handler must use GET (not POST) for OpenAI content download.""" from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -1431,5 +1509,117 @@ class TestVideoEndpointsProxyLitellmParams: ) +def test_video_remix_handler_uses_api_key_from_litellm_params(): + """Sync remix handler should fall back to litellm_params api_key when api_key param is None.""" + handler = BaseLLMHTTPHandler() + config = OpenAIVideoConfig() + + with patch.object(config, "validate_environment") as mock_validate: + mock_validate.return_value = {"Authorization": "Bearer deployment-key"} + + with patch.object(config, "transform_video_remix_request") as mock_transform: + mock_transform.return_value = ("https://api.openai.com/v1/videos/video_123/remix", {"prompt": "remix it"}) + + with patch.object(config, "transform_video_remix_response") as mock_resp: + mock_resp.return_value = MagicMock() + + mock_client = MagicMock() + mock_client.post.return_value = MagicMock(status_code=200) + + with patch( + "litellm.llms.custom_httpx.llm_http_handler._get_httpx_client", + return_value=mock_client, + ): + handler.video_remix_handler( + video_id="video_123", + prompt="remix it", + video_remix_provider_config=config, + custom_llm_provider="openai", + litellm_params={"api_key": "deployment-key", "api_base": "https://api.openai.com/v1"}, + logging_obj=MagicMock(), + timeout=5.0, + api_key=None, + _is_async=False, + ) + + mock_validate.assert_called_once() + assert mock_validate.call_args.kwargs["api_key"] == "deployment-key" + + +@pytest.mark.asyncio +async def test_async_video_remix_handler_uses_api_key_from_litellm_params(): + """Async remix handler should fall back to litellm_params api_key when api_key param is None.""" + handler = BaseLLMHTTPHandler() + config = OpenAIVideoConfig() + + with patch.object(config, "validate_environment") as mock_validate: + mock_validate.return_value = {"Authorization": "Bearer deployment-key"} + + with patch.object(config, "transform_video_remix_request") as mock_transform: + mock_transform.return_value = ("https://api.openai.com/v1/videos/video_123/remix", {"prompt": "remix it"}) + + with patch.object(config, "transform_video_remix_response") as mock_resp: + mock_resp.return_value = MagicMock() + + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_response = MagicMock(status_code=200) + mock_client.post = AsyncMock(return_value=mock_response) + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client", + return_value=mock_client, + ): + await handler.async_video_remix_handler( + video_id="video_123", + prompt="remix it", + video_remix_provider_config=config, + custom_llm_provider="openai", + litellm_params={"api_key": "deployment-key", "api_base": "https://api.openai.com/v1"}, + logging_obj=MagicMock(), + timeout=5.0, + api_key=None, + ) + + mock_validate.assert_called_once() + assert mock_validate.call_args.kwargs["api_key"] == "deployment-key" + + +def test_video_remix_handler_prefers_explicit_api_key(): + """Sync remix handler should prefer explicit api_key over litellm_params.""" + handler = BaseLLMHTTPHandler() + config = OpenAIVideoConfig() + + with patch.object(config, "validate_environment") as mock_validate: + mock_validate.return_value = {"Authorization": "Bearer explicit-key"} + + with patch.object(config, "transform_video_remix_request") as mock_transform: + mock_transform.return_value = ("https://api.openai.com/v1/videos/video_123/remix", {"prompt": "remix it"}) + + with patch.object(config, "transform_video_remix_response") as mock_resp: + mock_resp.return_value = MagicMock() + + mock_client = MagicMock() + mock_client.post.return_value = MagicMock(status_code=200) + + with patch( + "litellm.llms.custom_httpx.llm_http_handler._get_httpx_client", + return_value=mock_client, + ): + handler.video_remix_handler( + video_id="video_123", + prompt="remix it", + video_remix_provider_config=config, + custom_llm_provider="openai", + litellm_params={"api_key": "deployment-key", "api_base": "https://api.openai.com/v1"}, + logging_obj=MagicMock(), + timeout=5.0, + api_key="explicit-key", + _is_async=False, + ) + + mock_validate.assert_called_once() + assert mock_validate.call_args.kwargs["api_key"] == "explicit-key" + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 164368eb6b..b05d707d5a 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "dev": "next dev", + "dev:webpack": "next dev --webpack", "build": "next build", "start": "next start", "lint": "next lint", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/blogPosts/useBlogPosts.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/blogPosts/useBlogPosts.ts new file mode 100644 index 0000000000..81d55e8765 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/blogPosts/useBlogPosts.ts @@ -0,0 +1,32 @@ +import { getProxyBaseUrl } from "@/components/networking"; +import { useQuery } from "@tanstack/react-query"; + +export interface BlogPost { + title: string; + description: string; + date: string; + url: string; +} + +export interface BlogPostsResponse { + posts: BlogPost[]; +} + +async function fetchBlogPosts(): Promise { + const baseUrl = getProxyBaseUrl(); + const response = await fetch(`${baseUrl}/public/litellm_blog_posts`); + if (!response.ok) { + throw new Error(`Failed to fetch blog posts: ${response.statusText}`); + } + return response.json(); +} + +export const useBlogPosts = () => { + return useQuery({ + queryKey: ["blogPosts"], + queryFn: fetchBlogPosts, + staleTime: 60 * 60 * 1000, + retry: 1, + retryDelay: 0, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableBlogPosts.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableBlogPosts.ts new file mode 100644 index 0000000000..a7b37b78d4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableBlogPosts.ts @@ -0,0 +1,33 @@ +import { LOCAL_STORAGE_EVENT, getLocalStorageItem } from "@/utils/localStorageUtils"; +import { useSyncExternalStore } from "react"; + +function subscribe(callback: () => void) { + const onStorage = (e: StorageEvent) => { + if (e.key === "disableBlogPosts") { + callback(); + } + }; + + const onCustom = (e: Event) => { + const { key } = (e as CustomEvent).detail; + if (key === "disableBlogPosts") { + callback(); + } + }; + + window.addEventListener("storage", onStorage); + window.addEventListener(LOCAL_STORAGE_EVENT, onCustom); + + return () => { + window.removeEventListener("storage", onStorage); + window.removeEventListener(LOCAL_STORAGE_EVENT, onCustom); + }; +} + +function getSnapshot() { + return getLocalStorageItem("disableBlogPosts") === "true"; +} + +export function useDisableBlogPosts() { + return useSyncExternalStore(subscribe, getSnapshot); +} diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index ae3bd76e3c..fb749d7afb 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -13,6 +13,7 @@ import { fetchTeams } from "@/components/common_components/fetch_teams"; import LoadingScreen from "@/components/common_components/LoadingScreen"; import { CostTrackingSettings } from "@/components/CostTrackingSettings"; import GeneralSettings from "@/components/general_settings"; +import GuardrailsMonitorView from "@/components/GuardrailsMonitor/GuardrailsMonitorView"; import GuardrailsPanel from "@/components/guardrails"; import PoliciesPanel from "@/components/policies"; import { Team } from "@/components/key_team_helpers/key_list"; @@ -547,6 +548,8 @@ function CreateKeyPageContent() { ) : page == "vector-stores" ? ( + ) : page == "guardrails-monitor" ? ( + ) : page == "new_usage" ? ( = ({ accessToken, publicPage, client = openai.OpenAI( api_key="your_api_key", - base_url="http://0.0.0.0:4000" # Your LiteLLM Proxy URL + base_url="${getProxyBaseUrl()}" # Your LiteLLM Proxy URL ) response = client.chat.completions.create( @@ -997,7 +997,7 @@ import asyncio config = { "mcpServers": { "${selectedMcpServer.server_name}": { - "url": "http://localhost:4000/${selectedMcpServer.server_name}/mcp", + "url": "${getProxyBaseUrl()}/${selectedMcpServer.server_name}/mcp", "headers": { "x-litellm-api-key": "Bearer sk-1234" } @@ -1016,7 +1016,7 @@ async def main(): # Call a tool response = await client.call_tool( - name="tool_name", + name="tool_name", arguments={"arg": "value"} ) print(f"Response: {response}") diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/EvaluationSettingsModal.tsx new file mode 100644 index 0000000000..f502d2a8a3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/EvaluationSettingsModal.tsx @@ -0,0 +1,157 @@ +import { CloseOutlined, PlayCircleOutlined } from "@ant-design/icons"; +import { Button, Modal, Select, Input } from "antd"; +import React, { useEffect, useState } from "react"; +import { fetchAvailableModels, type ModelGroup } from "@/components/playground/llm_calls/fetch_models"; + +const DEFAULT_PROMPT = `Evaluate whether this guardrail's decision was correct. +Analyze the user input, the guardrail action taken, and determine if it was appropriate. + +Consider: +β€” Was the user's intent genuinely harmful or policy-violating? +β€” Was the guardrail's action (block / flag / pass) appropriate? +β€” Could this be a false positive or false negative? + +Return a structured verdict with confidence and justification.`; + +const DEFAULT_SCHEMA = `{ + "verdict": "correct" | "false_positive" | "false_negative", + "confidence": 0.0, + "justification": "string", + "risk_category": "string", + "suggested_action": "keep" | "adjust threshold" | "add allowlist" +} +`; + +export interface EvaluationSettingsModalProps { + open: boolean; + onClose: () => void; + guardrailName?: string; + accessToken: string | null; + onRunEvaluation?: (settings: { prompt: string; schema: string; model: string }) => void; +} + +export function EvaluationSettingsModal({ + open, + onClose, + guardrailName, + accessToken, + onRunEvaluation, +}: EvaluationSettingsModalProps) { + const [prompt, setPrompt] = useState(DEFAULT_PROMPT); + const [schema, setSchema] = useState(DEFAULT_SCHEMA); + const [model, setModel] = useState(null); + const [modelOptions, setModelOptions] = useState([]); + const [loadingModels, setLoadingModels] = useState(false); + + useEffect(() => { + if (!open || !accessToken) { + setModelOptions([]); + return; + } + let cancelled = false; + setLoadingModels(true); + fetchAvailableModels(accessToken) + .then((list) => { + if (!cancelled) setModelOptions(list); + }) + .catch(() => { + if (!cancelled) setModelOptions([]); + }) + .finally(() => { + if (!cancelled) setLoadingModels(false); + }); + return () => { + cancelled = true; + }; + }, [open, accessToken]); + + const handleResetPrompt = () => setPrompt(DEFAULT_PROMPT); + const handleRun = () => { + if (model) { + onRunEvaluation?.({ prompt, schema, model }); + onClose(); + } + }; + + const modelSelectOptions = modelOptions.map((m) => ({ + value: m.model_group, + label: m.model_group, + })); + + return ( + } + destroyOnClose + > +

+ {guardrailName + ? `Configure AI evaluation for ${guardrailName}` + : "Configure AI evaluation for re-running on logs"} +

+ +
+
+
+ + +
+ setPrompt(e.target.value)} + rows={6} + className="font-mono text-sm" + /> +

+ System prompt sent to the evaluation model. Output is structured via response_format. +

+
+ +
+ +

response_format: json_schema

+ setSchema(e.target.value)} + rows={6} + className="font-mono text-sm" + /> +
+ +
+ + ({ value: v.id, label: v.label }))} + style={{ width: 140 }} + /> + +
+
+ + +
+
+ + {showVersionHistory && ( +
+ {versions.map((v) => ( +
+
+ + {v.id} + + {v.changes} +
+
+ {v.author} + {v.date} +
+
+ ))} +
+ )} + + + {/* Parameters */} +
+

Parameters

+

Configure {guardrailName} behavior

+ +
+
+ + +
+ +
+ + +
+ +
+ + Guardrail enabled in production +
+
+
+ + {/* Custom Code Override */} +
+
+
+

+ + Custom Code Override +

+

+ Replace the built-in guardrail with custom evaluation code +

+
+ +
+ + {useCustomCode && ( + setCustomCode(e.target.value)} + placeholder={`async def evaluate(input_text: str, context: dict) -> dict: + # Return {"score": 0.0-1.0, "passed": bool, "reason": str} + # Example: + if "banned_word" in input_text.lower(): + return {"score": 0.1, "passed": False, "reason": "Banned word detected"} + return {"score": 0.9, "passed": True, "reason": "No violations"}`} + rows={10} + className="font-mono text-sm" + /> + )} +
+ + {/* Re-run on Failing Logs */} +
+

Test Configuration

+

+ Re-run this guardrail on recent failing logs to validate your changes +

+ +
+ + + {rerunStatus === "success" && ( + + 7/10 would now pass with new config + + )} + + {rerunStatus === "error" && ( + Error running tests + )} +
+
+ + ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailDetail.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailDetail.tsx new file mode 100644 index 0000000000..3447b4cb78 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailDetail.tsx @@ -0,0 +1,245 @@ +import { + ArrowLeftOutlined, + SafetyOutlined, + SettingOutlined, + WarningOutlined, +} from "@ant-design/icons"; +import { useQuery } from "@tanstack/react-query"; +import { Col, Grid } from "@tremor/react"; +import { Button, Spin, Tabs } from "antd"; +import React, { useMemo, useState } from "react"; +import { + getGuardrailsUsageDetail, + getGuardrailsUsageLogs, +} from "@/components/networking"; +import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; +import { LogViewer } from "./LogViewer"; +import { MetricCard } from "./MetricCard"; +import type { LogEntry } from "./mockData"; + +interface GuardrailDetailProps { + guardrailId: string; + onBack: () => void; + accessToken?: string | null; + startDate: string; + endDate: string; +} + +const statusColors: Record< + string, + { bg: string; text: string; dot: string } +> = { + healthy: { bg: "bg-green-50", text: "text-green-700", dot: "bg-green-500" }, + warning: { bg: "bg-amber-50", text: "text-amber-700", dot: "bg-amber-500" }, + critical: { bg: "bg-red-50", text: "text-red-700", dot: "bg-red-500" }, +}; + +export function GuardrailDetail({ + guardrailId, + onBack, + accessToken = null, + startDate, + endDate, +}: GuardrailDetailProps) { + const [activeTab, setActiveTab] = useState("overview"); + const [evaluationModalOpen, setEvaluationModalOpen] = useState(false); + const [logsPage, setLogsPage] = useState(1); + const logsPageSize = 50; + + const { data: detailData, isLoading: detailLoading, error: detailError } = useQuery({ + queryKey: ["guardrails-usage-detail", guardrailId, startDate, endDate], + queryFn: () => getGuardrailsUsageDetail(accessToken!, guardrailId, startDate, endDate), + enabled: !!accessToken && !!guardrailId, + }); + const { data: logsData, isLoading: logsLoading } = useQuery({ + queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize], + queryFn: () => + getGuardrailsUsageLogs(accessToken!, { + guardrailId, + page: logsPage, + pageSize: logsPageSize, + startDate, + endDate, + }), + enabled: !!accessToken && !!guardrailId, + }); + + const logs: LogEntry[] = useMemo(() => { + const list = logsData?.logs ?? []; + return list.map((l: Record) => ({ + id: l.id as string, + timestamp: l.timestamp as string, + action: l.action as "blocked" | "passed" | "flagged", + score: l.score as number | undefined, + model: l.model as string | undefined, + input_snippet: l.input_snippet as string | undefined, + output_snippet: l.output_snippet as string | undefined, + reason: l.reason as string | undefined, + })); + }, [logsData?.logs]); + + const data = detailData + ? { + name: detailData.guardrail_name, + description: detailData.description ?? "", + status: detailData.status, + provider: detailData.provider, + type: detailData.type, + requestsEvaluated: detailData.requestsEvaluated, + failRate: detailData.failRate, + avgScore: detailData.avgScore, + avgLatency: detailData.avgLatency, + } + : { + name: guardrailId, + description: "", + status: "healthy", + provider: "β€”", + type: "β€”", + requestsEvaluated: 0, + failRate: 0, + avgScore: undefined as number | undefined, + avgLatency: undefined as number | undefined, + }; + const statusStyle = statusColors[data.status] ?? statusColors.healthy; + + if (detailLoading && !detailData) { + return ( +
+ +
+ ); + } + if (detailError && !detailData) { + return ( +
+ +

Failed to load guardrail details.

+
+ ); + } + + return ( +
+
+ + +
+
+
+ +

{data.name}

+ + + {data.status.charAt(0).toUpperCase() + data.status.slice(1)} + +
+

{data.description}

+
+
+ + {data.provider} + +
+
+
+ + + + {activeTab === "overview" && ( +
+ + + + + + 15 ? "text-red-600" : data.failRate > 5 ? "text-amber-600" : "text-green-600" + } + subtitle={`${Math.round((data.requestsEvaluated * data.failRate) / 100).toLocaleString()} blocked`} + icon={data.failRate > 15 ? : undefined} + /> + + + 150 + ? "text-red-600" + : data.avgLatency > 50 + ? "text-amber-600" + : "text-green-600" + : "text-gray-500" + } + subtitle={data.avgLatency != null ? "Per request (avg)" : "No data"} + /> + + + + +
+ )} + + {activeTab === "logs" && ( +
+ +
+ )} + + setEvaluationModalOpen(false)} + guardrailName={data.name} + accessToken={accessToken} + /> +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.test.tsx new file mode 100644 index 0000000000..081e29ec9e --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.test.tsx @@ -0,0 +1,52 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { describe, expect, it, vi } from "vitest"; +import GuardrailsMonitorView from "./GuardrailsMonitorView"; +import * as networking from "@/components/networking"; + +vi.mock("@/components/networking", () => ({ + getGuardrailsUsageOverview: vi.fn(), + formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), +})); + +const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); + +function wrapper({ children }: { children: React.ReactNode }) { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + }, + }); + return ( + + {children} + + ); +} + +describe("GuardrailsMonitorView", () => { + it("should render overview and fetch guardrails usage when accessToken is provided", async () => { + mockGetGuardrailsUsageOverview.mockResolvedValue({ + rows: [], + chart: [], + totalRequests: 0, + totalBlocked: 0, + passRate: 100, + }); + + render( + , + { wrapper } + ); + + expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeDefined(); + await waitFor(() => { + expect(mockGetGuardrailsUsageOverview).toHaveBeenCalled(); + }); + }); + + it("should render without crashing when accessToken is null", async () => { + render(, { wrapper }); + expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeDefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.tsx new file mode 100644 index 0000000000..7214b0642b --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsMonitorView.tsx @@ -0,0 +1,74 @@ +import type { DateRangePickerValue } from "@tremor/react"; +import React, { useCallback, useMemo, useState } from "react"; +import { formatDate } from "@/components/networking"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { GuardrailDetail } from "./GuardrailDetail"; +import { GuardrailsOverview } from "./GuardrailsOverview"; + +type View = + | { type: "overview" } + | { type: "detail"; guardrailId: string }; + +interface GuardrailsMonitorViewProps { + accessToken?: string | null; +} + +const defaultEnd = new Date(); +const defaultStart = new Date(); +defaultStart.setDate(defaultStart.getDate() - 7); + +export default function GuardrailsMonitorView({ accessToken = null }: GuardrailsMonitorViewProps) { + const [view, setView] = useState({ type: "overview" }); + + const initialFrom = useMemo(() => new Date(defaultStart), []); + const initialTo = useMemo(() => new Date(defaultEnd), []); + + const [dateValue, setDateValue] = useState({ + from: initialFrom, + to: initialTo, + }); + + const startDate = dateValue.from ? formatDate(dateValue.from) : ""; + const endDate = dateValue.to ? formatDate(dateValue.to) : ""; + + const handleDateChange = useCallback((newValue: DateRangePickerValue) => { + setDateValue(newValue); + }, []); + + const handleSelectGuardrail = (id: string) => { + setView({ type: "detail", guardrailId: id }); + }; + + const handleBack = () => { + setView({ type: "overview" }); + }; + + return ( +
+
+ +
+ {view.type === "overview" ? ( + + ) : ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsOverview.tsx new file mode 100644 index 0000000000..d2fa53bc6c --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailsOverview.tsx @@ -0,0 +1,313 @@ +import { + DownloadOutlined, + RiseOutlined, + SafetyOutlined, + SettingOutlined, + WarningOutlined, +} from "@ant-design/icons"; +import { useQuery } from "@tanstack/react-query"; +import { Card, Col, Grid, Title } from "@tremor/react"; +import { Button, Spin, Table } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useMemo, useState } from "react"; +import { getGuardrailsUsageOverview } from "@/components/networking"; +import { type PerformanceRow } from "./mockData"; +import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; +import { MetricCard } from "./MetricCard"; +import { ScoreChart } from "./ScoreChart"; + +interface GuardrailsOverviewProps { + accessToken?: string | null; + startDate: string; + endDate: string; + onSelectGuardrail: (id: string) => void; +} + +type SortKey = + | "failRate" + | "requestsEvaluated" + | "avgLatency" + | "falsePositiveRate" + | "falseNegativeRate"; + +const providerColors: Record = { + Bedrock: "bg-orange-100 text-orange-700 border-orange-200", + "Google Cloud": "bg-sky-100 text-sky-700 border-sky-200", + LiteLLM: "bg-indigo-100 text-indigo-700 border-indigo-200", + Custom: "bg-gray-100 text-gray-600 border-gray-200", +}; + +function computeMetricsFromRows(data: PerformanceRow[]) { + const totalRequests = data.reduce((sum, r) => sum + r.requestsEvaluated, 0); + const totalBlocked = data.reduce( + (sum, r) => sum + Math.round((r.requestsEvaluated * r.failRate) / 100), + 0 + ); + const passRate = + totalRequests > 0 ? ((1 - totalBlocked / totalRequests) * 100).toFixed(1) : "0"; + const withLat = data.filter((r) => r.avgLatency != null); + const avgLatency = + withLat.length > 0 + ? Math.round(withLat.reduce((sum, r) => sum + (r.avgLatency ?? 0), 0) / withLat.length) + : 0; + return { totalRequests, totalBlocked, passRate, avgLatency, count: data.length }; +} + +export function GuardrailsOverview({ + accessToken = null, + startDate, + endDate, + onSelectGuardrail, +}: GuardrailsOverviewProps) { + const [sortBy, setSortBy] = useState("failRate"); + const [sortDir, setSortDir] = useState<"asc" | "desc">("desc"); + const [evaluationModalOpen, setEvaluationModalOpen] = useState(false); + + const { data: guardrailsData, isLoading: guardrailsLoading, error: guardrailsError } = useQuery({ + queryKey: ["guardrails-usage-overview", startDate, endDate], + queryFn: () => getGuardrailsUsageOverview(accessToken!, startDate, endDate), + enabled: !!accessToken, + }); + + const activeData: PerformanceRow[] = guardrailsData?.rows ?? []; + const metrics = useMemo(() => { + if (guardrailsData) { + return { + totalRequests: guardrailsData.totalRequests ?? 0, + totalBlocked: guardrailsData.totalBlocked ?? 0, + passRate: String(guardrailsData.passRate ?? 0), + avgLatency: activeData.length ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) : 0, + count: activeData.length, + }; + } + return computeMetricsFromRows(activeData); + }, [guardrailsData, activeData]); + const chartData = guardrailsData?.chart; + const sorted = useMemo(() => { + return [...activeData].sort((a, b) => { + const mult = sortDir === "desc" ? -1 : 1; + const aVal = a[sortBy] ?? 0; + const bVal = b[sortBy] ?? 0; + return (Number(aVal) - Number(bVal)) * mult; + }); + }, [activeData, sortBy, sortDir]); + const isLoading = guardrailsLoading; + const error = guardrailsError; + + const columns: ColumnsType = [ + { + title: "Guardrail", + dataIndex: "name", + key: "name", + render: (name: string, row) => ( + + ), + }, + { + title: "Provider", + dataIndex: "provider", + key: "provider", + render: (provider: string) => ( + + {provider} + + ), + }, + { + title: "Requests", + dataIndex: "requestsEvaluated", + key: "requestsEvaluated", + align: "right", + sorter: true, + sortOrder: sortBy === "requestsEvaluated" ? (sortDir === "desc" ? "descend" : "ascend") : null, + render: (v: number) => v.toLocaleString(), + }, + { + title: "Fail Rate", + dataIndex: "failRate", + key: "failRate", + align: "right", + sorter: true, + sortOrder: sortBy === "failRate" ? (sortDir === "desc" ? "descend" : "ascend") : null, + render: (v: number, row) => ( + 15 ? "text-red-600" : v > 5 ? "text-amber-600" : "text-green-600" + } + > + {v}% + {row.trend === "up" && ↑} + {row.trend === "down" && ↓} + + ), + }, + { + title: "Avg. latency added", + dataIndex: "avgLatency", + key: "avgLatency", + align: "right", + sorter: true, + sortOrder: sortBy === "avgLatency" ? (sortDir === "desc" ? "descend" : "ascend") : null, + render: (v?: number) => ( + 150 ? "text-red-600" : v > 50 ? "text-amber-600" : "text-green-600" + } + > + {v != null ? `${v}ms` : "β€”"} + + ), + }, + { + title: "Status", + dataIndex: "status", + key: "status", + align: "center", + render: (status: string) => ( + + + {status} + + ), + }, + ]; + + const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency"]; + const handleTableChange = (_pagination: unknown, _filters: unknown, sorter: unknown) => { + const s = sorter as { field?: keyof PerformanceRow; order?: string }; + if (s?.field && sortableKeys.includes(s.field as SortKey)) { + setSortBy(s.field as SortKey); + setSortDir(s.order === "ascend" ? "asc" : "desc"); + } + }; + + return ( +
+
+
+
+ +

Guardrails Monitor

+
+

+ Monitor guardrail performance across all requests +

+
+
+ +
+
+ + + + + + + } + /> + + + } + /> + + + 150 + ? "text-red-600" + : metrics.avgLatency > 50 + ? "text-amber-600" + : "text-green-600" + } + /> + + + + + + +
+ +
+ + + {(isLoading || error) && ( +
+ {isLoading && } + {error && Failed to load data. Try again.} +
+ )} +
+
+ + Guardrail Performance + +

+ Click a guardrail to view details, logs, and configuration +

+
+
+
+
+ ({ + onClick: () => onSelectGuardrail(row.id), + style: { cursor: "pointer" }, + })} + /> + + + setEvaluationModalOpen(false)} + accessToken={accessToken} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx new file mode 100644 index 0000000000..aed19ddfc8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -0,0 +1,227 @@ +import { + CheckCircleOutlined, + CloseOutlined, + DownOutlined, + WarningOutlined, +} from "@ant-design/icons"; +import { useQuery } from "@tanstack/react-query"; +import moment from "moment"; +import { Button, Spin } from "antd"; +import React, { useState } from "react"; +import { uiSpendLogsCall } from "@/components/networking"; +import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer"; +import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns"; +import type { LogEntry } from "./mockData"; + +const actionConfig: Record< + "blocked" | "passed" | "flagged", + { icon: React.ElementType; color: string; bg: string; border: string; label: string } +> = { + blocked: { + icon: CloseOutlined, + color: "text-red-600", + bg: "bg-red-50", + border: "border-red-200", + label: "Blocked", + }, + passed: { + icon: CheckCircleOutlined, + color: "text-green-600", + bg: "bg-green-50", + border: "border-green-200", + label: "Passed", + }, + flagged: { + icon: WarningOutlined, + color: "text-amber-600", + bg: "bg-amber-50", + border: "border-amber-200", + label: "Flagged", + }, +}; + +interface LogViewerProps { + guardrailName?: string; + filterAction?: "all" | "blocked" | "passed" | "flagged"; + logs?: LogEntry[]; + logsLoading?: boolean; + totalLogs?: number; + accessToken?: string | null; + startDate?: string; + endDate?: string; +} + +export function LogViewer({ + guardrailName, + filterAction = "all", + logs = [], + logsLoading = false, + totalLogs, + accessToken = null, + startDate = "", + endDate = "", +}: LogViewerProps) { + const [sampleSize, setSampleSize] = useState(10); + const [activeFilter, setActiveFilter] = useState(filterAction); + const [selectedRequestId, setSelectedRequestId] = useState(null); + const [drawerOpen, setDrawerOpen] = useState(false); + + const filteredLogs = logs.filter( + (log) => activeFilter === "all" || log.action === activeFilter + ); + const displayLogs = filteredLogs.slice(0, sampleSize); + const total = totalLogs ?? logs.length; + const sampleSizes = [10, 50, 100]; + const filters: Array<"all" | "blocked" | "flagged" | "passed"> = [ + "all", + "blocked", + "flagged", + "passed", + ]; + + const startTime = startDate + ? moment(startDate).utc().format("YYYY-MM-DD HH:mm:ss") + : moment().subtract(24, "hours").utc().format("YYYY-MM-DD HH:mm:ss"); + const endTime = endDate + ? moment(endDate).utc().endOf("day").format("YYYY-MM-DD HH:mm:ss") + : moment().utc().format("YYYY-MM-DD HH:mm:ss"); + + const { data: fullLogResponse } = useQuery({ + queryKey: ["spend-log-by-request", selectedRequestId, startTime, endTime], + queryFn: async () => { + if (!accessToken || !selectedRequestId) return null; + const res = await uiSpendLogsCall({ + accessToken, + start_date: startTime, + end_date: endTime, + page: 1, + page_size: 10, + params: { request_id: selectedRequestId }, + }); + return res as { data: ViewLogsLogEntry[]; total: number }; + }, + enabled: Boolean(accessToken && selectedRequestId && drawerOpen), + }); + + const selectedLog: ViewLogsLogEntry | null = + fullLogResponse?.data?.[0] ?? null; + + const handleLogClick = (log: LogEntry) => { + setSelectedRequestId(log.id); + setDrawerOpen(true); + }; + + const handleCloseDrawer = () => { + setDrawerOpen(false); + setSelectedRequestId(null); + }; + + return ( +
+
+
+
+

+ {guardrailName ? `Logs β€” ${guardrailName}` : "Request Logs"} +

+

+ {logsLoading + ? "Loading…" + : logs.length > 0 + ? `Showing ${displayLogs.length} of ${total} entries` + : "No logs for this period. Select a guardrail and date range."} +

+
+ {logs.length > 0 && ( +
+
+ {filters.map((f) => ( + + ))} +
+
+
+ Sample: + {sampleSizes.map((size) => ( + + ))} +
+
+ )} +
+
+ + {logsLoading && ( +
+ +
+ )} + {!logsLoading && displayLogs.length === 0 && ( +
+ No logs to display. Adjust filters or date range. +
+ )} + {!logsLoading && displayLogs.length > 0 && ( +
+ {displayLogs.map((log) => { + const config = actionConfig[log.action]; + const ActionIcon = config.icon; + return ( + + ); + })} +
+ )} + + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx new file mode 100644 index 0000000000..4a11efe72a --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -0,0 +1,30 @@ +import React, { type ReactNode } from "react"; + +interface MetricCardProps { + label: string; + value: string | number; + valueColor?: string; + icon?: ReactNode; + subtitle?: string; +} + +export function MetricCard({ + label, + value, + valueColor = "text-gray-900", + icon, + subtitle, +}: MetricCardProps) { + return ( +
+
+ {label} + {icon && {icon}} +
+
+ {value} +
+ {subtitle &&

{subtitle}

} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.tsx new file mode 100644 index 0000000000..e4803747d4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.tsx @@ -0,0 +1,39 @@ +import { BarChart, Card, Title } from "@tremor/react"; +import React from "react"; + +/** + * Overview chart: Request Outcomes Over Time (passed vs blocked). + * Uses Tremor BarChart with stacked data. Data from usage/overview API (chart array). + */ +interface ScoreChartProps { + data?: Array<{ date: string; passed: number; blocked: number }>; +} + +export function ScoreChart({ data }: ScoreChartProps) { + const chartData = data && data.length > 0 ? data : []; + return ( + + + Request Outcomes Over Time + +
+ {chartData.length > 0 ? ( + v.toLocaleString()} + yAxisWidth={48} + showLegend={true} + stack={true} + /> + ) : ( +
+ No chart data for this period +
+ )} +
+ + ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts new file mode 100644 index 0000000000..7d99ebe7c4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -0,0 +1,50 @@ +/** + * Types for Guardrails Monitor dashboard (data from usage API). + */ + +export interface PerformanceRow { + id: string; + name: string; + type: string; + provider: string; + requestsEvaluated: number; + failRate: number; + avgScore?: number; + avgLatency?: number; + p95Latency?: number; + falsePositiveRate?: number; + falseNegativeRate?: number; + status: "healthy" | "warning" | "critical"; + trend: "up" | "down" | "stable"; +} + +export interface GuardrailDetailRecord { + name: string; + type: string; + provider: string; + requestsEvaluated: number; + failRate: number; + avgScore?: number; + avgLatency?: number; + p95Latency?: number; + falsePositiveRate?: number; + falsePositiveCount?: number; + falseNegativeRate?: number; + falseNegativeCount?: number; + status: string; + description: string; +} + +export interface LogEntry { + id: string; + timestamp: string; + input?: string; + output?: string; + input_snippet?: string; + output_snippet?: string; + score?: number; + action: "blocked" | "passed" | "flagged"; + model?: string; + reason?: string; + latency_ms?: number; +} diff --git a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx new file mode 100644 index 0000000000..4ca0aa2aae --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx @@ -0,0 +1,230 @@ +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils"; +import { BlogDropdown } from "./BlogDropdown"; + +let mockDisableBlogPosts = false; +let mockRefetch = vi.fn(); +let mockUseBlogPostsResult: { + data: { posts: { title: string; date: string; description: string; url: string }[] } | null | undefined; + isLoading: boolean; + isError: boolean; + refetch: () => void; +} = { + data: undefined, + isLoading: false, + isError: false, + refetch: mockRefetch, +}; + +vi.mock("@/app/(dashboard)/hooks/useDisableBlogPosts", () => ({ + useDisableBlogPosts: () => mockDisableBlogPosts, +})); + +vi.mock("@/app/(dashboard)/hooks/blogPosts/useBlogPosts", () => ({ + useBlogPosts: () => mockUseBlogPostsResult, +})); + +const MOCK_POSTS = [ + { title: "Post One", date: "2026-02-01", description: "Description one", url: "https://example.com/1" }, + { title: "Post Two", date: "2026-02-02", description: "Description two", url: "https://example.com/2" }, + { title: "Post Three", date: "2026-02-03", description: "Description three", url: "https://example.com/3" }, + { title: "Post Four", date: "2026-02-04", description: "Description four", url: "https://example.com/4" }, + { title: "Post Five", date: "2026-02-05", description: "Description five", url: "https://example.com/5" }, + { title: "Post Six", date: "2026-02-06", description: "Description six", url: "https://example.com/6" }, +]; + +async function openDropdown() { + const user = userEvent.setup(); + await user.hover(screen.getByRole("button", { name: /blog/i })); +} + +describe("BlogDropdown", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDisableBlogPosts = false; + mockRefetch = vi.fn(); + mockUseBlogPostsResult = { + data: undefined, + isLoading: false, + isError: false, + refetch: mockRefetch, + }; + }); + + describe("when blog posts are disabled", () => { + it("should render nothing", () => { + mockDisableBlogPosts = true; + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + }); + + describe("when blog posts are enabled", () => { + it("should render the Blog trigger button", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /blog/i })).toBeInTheDocument(); + }); + + describe("loading state", () => { + it("should show a loading spinner", async () => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, isLoading: true }; + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(document.querySelector(".anticon-loading")).toBeInTheDocument(); + }); + }); + }); + + describe("error state", () => { + beforeEach(() => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, isError: true }; + }); + + it("should show an error message", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("Failed to load posts")).toBeInTheDocument(); + }); + }); + + it("should show a Retry button", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /retry/i })).toBeInTheDocument(); + }); + }); + + it("should call refetch when Retry is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.hover(screen.getByRole("button", { name: /blog/i })); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /retry/i })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /retry/i })); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); + }); + + describe("empty state", () => { + it("should show 'No posts available' when data is null", async () => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, data: null }; + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("No posts available")).toBeInTheDocument(); + }); + }); + + it("should show 'No posts available' when posts array is empty", async () => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, data: { posts: [] } }; + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("No posts available")).toBeInTheDocument(); + }); + }); + }); + + describe("with posts", () => { + beforeEach(() => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, data: { posts: MOCK_POSTS.slice(0, 3) } }; + }); + + it("should render post titles", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("Post One")).toBeInTheDocument(); + expect(screen.getByText("Post Two")).toBeInTheDocument(); + expect(screen.getByText("Post Three")).toBeInTheDocument(); + }); + }); + + it("should render post descriptions", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("Description one")).toBeInTheDocument(); + }); + }); + + it("should render post links with correct attributes", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + const link = screen.getByRole("link", { name: /post one/i }); + expect(link).toHaveAttribute("href", "https://example.com/1"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + }); + + it("should render formatted post dates", async () => { + mockUseBlogPostsResult = { + ...mockUseBlogPostsResult, + data: { posts: [{ title: "Date Post", date: "2026-02-15", description: "Desc", url: "https://example.com" }] }, + }; + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("Feb 15, 2026")).toBeInTheDocument(); + }); + }); + + it("should render the 'View all posts' link", async () => { + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + const viewAllLink = screen.getByRole("link", { name: /view all posts/i }); + expect(viewAllLink).toHaveAttribute("href", "https://docs.litellm.ai/blog"); + expect(viewAllLink).toHaveAttribute("target", "_blank"); + expect(viewAllLink).toHaveAttribute("rel", "noopener noreferrer"); + }); + }); + }); + + describe("post limit", () => { + it("should render at most 5 posts when more than 5 are provided", async () => { + mockUseBlogPostsResult = { ...mockUseBlogPostsResult, data: { posts: MOCK_POSTS } }; + renderWithProviders(); + + await openDropdown(); + + await waitFor(() => { + expect(screen.getByText("Post One")).toBeInTheDocument(); + expect(screen.getByText("Post Five")).toBeInTheDocument(); + expect(screen.queryByText("Post Six")).not.toBeInTheDocument(); + }); + }); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx new file mode 100644 index 0000000000..ddb2a33cda --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx @@ -0,0 +1,84 @@ +import { useDisableBlogPosts } from "@/app/(dashboard)/hooks/useDisableBlogPosts"; +import { useBlogPosts, type BlogPost } from "@/app/(dashboard)/hooks/blogPosts/useBlogPosts"; +import { LoadingOutlined } from "@ant-design/icons"; +import { Button, Dropdown, Space, Typography } from "antd"; +import type { MenuProps } from "antd"; +import React from "react"; + +const { Text, Title, Paragraph } = Typography; + +function formatDate(dateStr: string): string { + const date = new Date(dateStr + "T00:00:00"); + return date.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + }); +} + +export const BlogDropdown: React.FC = () => { + const disableBlogPosts = useDisableBlogPosts(); + + const { data, isLoading, isError, refetch } = useBlogPosts(); + + if (disableBlogPosts) { + return null; + } + + let items: MenuProps["items"]; + + if (isLoading) { + items = [{ key: "loading", label: , disabled: true }]; + } else if (isError) { + items = [ + { + key: "error", + label: ( + + Failed to load posts + + + ), + disabled: true, + }, + ]; + } else if (!data || data.posts.length === 0) { + items = [{ key: "empty", label: No posts available, disabled: true }]; + } else { + items = [ + ...data.posts.slice(0, 5).map((post: BlogPost) => ({ + key: post.url, + label: ( + + + {post.title} + + + {formatDate(post.date)} + + {post.description} + + ), + })), + { type: "divider" as const }, + { + key: "view-all", + label: ( + + View all posts + + ), + }, + ]; + } + + return ( + + + + ); +}; + +export default BlogDropdown; diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx index 90e02ae447..2bef9a8077 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx @@ -1,4 +1,5 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useDisableBlogPosts } from "@/app/(dashboard)/hooks/useDisableBlogPosts"; import { useDisableShowPrompts } from "@/app/(dashboard)/hooks/useDisableShowPrompts"; import { useDisableUsageIndicator } from "@/app/(dashboard)/hooks/useDisableUsageIndicator"; import { @@ -29,6 +30,7 @@ const UserDropdown: React.FC = ({ onLogout }) => { const { userId, userEmail, userRole, premiumUser } = useAuthorized(); const disableShowPrompts = useDisableShowPrompts(); const disableUsageIndicator = useDisableUsageIndicator(); + const disableBlogPosts = useDisableBlogPosts(); const [disableShowNewBadge, setDisableShowNewBadge] = useState(false); useEffect(() => { @@ -148,6 +150,23 @@ const UserDropdown: React.FC = ({ onLogout }) => { aria-label="Toggle hide usage indicator" /> + + Hide Blog Posts + { + if (checked) { + setLocalStorageItem("disableBlogPosts", "true"); + emitLocalStorageChange("disableBlogPosts"); + } else { + removeLocalStorageItem("disableBlogPosts"); + emitLocalStorageChange("disableBlogPosts"); + } + }} + aria-label="Toggle hide blog posts" + /> + ); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.test.tsx new file mode 100644 index 0000000000..2b9d9ba9f8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.test.tsx @@ -0,0 +1,165 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import MCPSemanticFilterSettings from "./MCPSemanticFilterSettings"; +import { useMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings"; +import { useUpdateMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings"; + +vi.mock( + "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings", + () => ({ useMCPSemanticFilterSettings: vi.fn() }) +); + +vi.mock( + "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings", + () => ({ useUpdateMCPSemanticFilterSettings: vi.fn() }) +); + +vi.mock("@/components/playground/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([]), +})); + +vi.mock("./MCPSemanticFilterTestPanel", () => ({ + default: () =>
, +})); + +vi.mock("./semanticFilterTestUtils", () => ({ + getCurlCommand: vi.fn().mockReturnValue("curl ..."), + runSemanticFilterTest: vi.fn(), +})); + +const mockMutate = vi.fn(); + +const defaultSettingsData = { + field_schema: { + properties: { + enabled: { description: "Enable semantic filtering for MCP tools" }, + }, + }, + values: { + enabled: false, + embedding_model: "text-embedding-3-small", + top_k: 10, + similarity_threshold: 0.3, + }, +}; + +// Helper that renders the component and flushes the fetchAvailableModels effect +async function renderSettings(props: React.ComponentProps) { + render(); + if (props.accessToken) { + // Let the async fetchAvailableModels effect settle to avoid act() warnings + await act(async () => {}); + } +} + +describe("MCPSemanticFilterSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(useMCPSemanticFilterSettings).mockReturnValue({ + data: defaultSettingsData, + isLoading: false, + isError: false, + error: null, + } as any); + vi.mocked(useUpdateMCPSemanticFilterSettings).mockReturnValue({ + mutate: mockMutate, + isPending: false, + error: null, + } as any); + }); + + it("should render", async () => { + await renderSettings({ accessToken: "test-token" }); + expect(screen.getByText("Semantic Tool Filtering")).toBeInTheDocument(); + }); + + it("should show a login prompt when accessToken is null", () => { + render(); + expect(screen.getByText(/please log in/i)).toBeInTheDocument(); + }); + + it("should not render the form when accessToken is null", () => { + render(); + expect(screen.queryByText("Enable Semantic Filtering")).not.toBeInTheDocument(); + }); + + it("should not show the settings content while loading", async () => { + vi.mocked(useMCPSemanticFilterSettings).mockReturnValue({ + data: undefined, + isLoading: true, + isError: false, + error: null, + } as any); + await renderSettings({ accessToken: "test-token" }); + expect(screen.queryByText("Semantic Tool Filtering")).not.toBeInTheDocument(); + }); + + it("should show an error alert when data fails to load", async () => { + vi.mocked(useMCPSemanticFilterSettings).mockReturnValue({ + data: undefined, + isLoading: false, + isError: true, + error: new Error("Network error"), + } as any); + await renderSettings({ accessToken: "test-token" }); + expect( + screen.getByText("Could not load MCP Semantic Filter settings") + ).toBeInTheDocument(); + expect(screen.getByText("Network error")).toBeInTheDocument(); + }); + + it("should show the error message from the error object when loading fails", async () => { + vi.mocked(useMCPSemanticFilterSettings).mockReturnValue({ + data: undefined, + isLoading: false, + isError: true, + error: new Error("Connection refused"), + } as any); + await renderSettings({ accessToken: "test-token" }); + expect(screen.getByText("Connection refused")).toBeInTheDocument(); + }); + + it("should render the info alert and form fields when data is loaded", async () => { + await renderSettings({ accessToken: "test-token" }); + expect(screen.getByText("Semantic Tool Filtering")).toBeInTheDocument(); + expect(screen.getByText("Enable Semantic Filtering")).toBeInTheDocument(); + expect(screen.getByText("Top K Results")).toBeInTheDocument(); + expect(screen.getByText("Similarity Threshold")).toBeInTheDocument(); + }); + + it("should render the test panel", async () => { + await renderSettings({ accessToken: "test-token" }); + expect(screen.getByTestId("mcp-test-panel")).toBeInTheDocument(); + }); + + it("should have Save Settings button disabled initially", async () => { + await renderSettings({ accessToken: "test-token" }); + expect( + screen.getByRole("button", { name: /save settings/i }) + ).toBeDisabled(); + }); + + it("should enable Save Settings button after a form field is changed", async () => { + const user = userEvent.setup(); + await renderSettings({ accessToken: "test-token" }); + + expect(screen.getByRole("button", { name: /save settings/i })).toBeDisabled(); + + await user.click(screen.getByRole("switch")); + + expect(screen.getByRole("button", { name: /save settings/i })).not.toBeDisabled(); + }); + + it("should show an error alert when the mutation fails", async () => { + vi.mocked(useUpdateMCPSemanticFilterSettings).mockReturnValue({ + mutate: mockMutate, + isPending: false, + error: new Error("Failed to update settings"), + } as any); + await renderSettings({ accessToken: "test-token" }); + expect(screen.getByText("Could not update settings")).toBeInTheDocument(); + expect(screen.getByText("Failed to update settings")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx new file mode 100644 index 0000000000..974a6a7bf0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx @@ -0,0 +1,141 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import MCPSemanticFilterTestPanel from "./MCPSemanticFilterTestPanel"; +import { TestResult } from "./semanticFilterTestUtils"; + +vi.mock("@/components/common_components/ModelSelector", () => ({ + default: ({ onChange, value, labelText, disabled }: any) => ( +
+ + +
+ ), +})); + +const buildProps = ( + overrides: Partial> = {} +) => ({ + accessToken: "test-token", + testQuery: "", + setTestQuery: vi.fn(), + testModel: "gpt-4o", + setTestModel: vi.fn(), + isTesting: false, + onTest: vi.fn(), + filterEnabled: true, + testResult: null as TestResult | null, + curlCommand: "curl --location 'http://localhost:4000/v1/responses'", + ...overrides, +}); + +describe("MCPSemanticFilterTestPanel", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render the Test Configuration card", () => { + render(); + expect(screen.getByText("Test Configuration")).toBeInTheDocument(); + }); + + it("should show the test query textarea", () => { + render(); + expect( + screen.getByPlaceholderText(/enter a test query to see which tools/i) + ).toBeInTheDocument(); + }); + + it("should call setTestQuery when user types in the query field", () => { + const mockSetTestQuery = vi.fn(); + render(); + + const textarea = screen.getByPlaceholderText(/enter a test query to see which tools/i); + fireEvent.change(textarea, { target: { value: "find relevant tools" } }); + + expect(mockSetTestQuery).toHaveBeenCalledWith("find relevant tools"); + }); + + it("should disable the Test Filter button when testQuery is empty", () => { + render(); + expect(screen.getByRole("button", { name: /test filter/i })).toBeDisabled(); + }); + + it("should disable the Test Filter button when filterEnabled is false", () => { + render( + + ); + expect(screen.getByRole("button", { name: /test filter/i })).toBeDisabled(); + }); + + it("should enable the Test Filter button when testQuery is set and filter is enabled", () => { + render( + + ); + expect(screen.getByRole("button", { name: /test filter/i })).not.toBeDisabled(); + }); + + it("should call onTest when the Test Filter button is clicked", async () => { + const mockOnTest = vi.fn(); + const user = userEvent.setup(); + render( + + ); + + await user.click(screen.getByRole("button", { name: /test filter/i })); + expect(mockOnTest).toHaveBeenCalledOnce(); + }); + + it("should show a warning when semantic filtering is disabled", () => { + render(); + expect(screen.getByText("Semantic filtering is disabled")).toBeInTheDocument(); + }); + + it("should not show the disabled warning when filterEnabled is true", () => { + render(); + expect(screen.queryByText("Semantic filtering is disabled")).not.toBeInTheDocument(); + }); + + it("should display test results when testResult is provided", () => { + const testResult: TestResult = { + totalTools: 10, + selectedTools: 3, + tools: ["wiki-fetch", "github-search", "slack-post"], + }; + render(); + + expect(screen.getByText("3 tools selected")).toBeInTheDocument(); + expect(screen.getByText("Filtered from 10 available tools")).toBeInTheDocument(); + expect(screen.getByText("wiki-fetch")).toBeInTheDocument(); + expect(screen.getByText("github-search")).toBeInTheDocument(); + expect(screen.getByText("slack-post")).toBeInTheDocument(); + }); + + it("should not render the results section when testResult is null", () => { + render(); + expect(screen.queryByText("Results")).not.toBeInTheDocument(); + }); + + it("should show the curl command in the API Usage tab", async () => { + const user = userEvent.setup(); + const curlCommand = "curl --location 'http://localhost:4000/v1/responses' --header 'Authorization: Bearer sk-1234'"; + render(); + + await user.click(screen.getByRole("tab", { name: "API Usage" })); + + expect(screen.getByText(curlCommand)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts new file mode 100644 index 0000000000..12acdf8b8b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { getCurlCommand, runSemanticFilterTest } from "./semanticFilterTestUtils"; +import { testMCPSemanticFilter } from "@/components/networking"; +import NotificationManager from "@/components/molecules/notifications_manager"; + +vi.mock("@/components/networking", () => ({ + testMCPSemanticFilter: vi.fn(), +})); + +describe("getCurlCommand", () => { + it("should include the model name in the curl command", () => { + const result = getCurlCommand("gpt-4o", "test query"); + expect(result).toContain('"gpt-4o"'); + }); + + it("should include the query in the curl command", () => { + const result = getCurlCommand("gpt-4o", "find relevant files"); + expect(result).toContain("find relevant files"); + }); + + it("should use a placeholder when query is empty", () => { + const result = getCurlCommand("gpt-4o", ""); + expect(result).toContain("Your query here"); + }); +}); + +describe("runSemanticFilterTest", () => { + const mockSetIsTesting = vi.fn(); + const mockSetTestResult = vi.fn(); + const baseArgs = { + accessToken: "test-token", + testModel: "gpt-4o", + testQuery: "find relevant files", + setIsTesting: mockSetIsTesting, + setTestResult: mockSetTestResult, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should call NotificationManager.error and not set isTesting when testQuery is empty", async () => { + await runSemanticFilterTest({ ...baseArgs, testQuery: "" }); + expect(NotificationManager.error).toHaveBeenCalledWith("Please enter a query and select a model"); + expect(mockSetIsTesting).not.toHaveBeenCalled(); + }); + + it("should call NotificationManager.error and not set isTesting when testModel is empty", async () => { + await runSemanticFilterTest({ ...baseArgs, testModel: "" }); + expect(NotificationManager.error).toHaveBeenCalledWith("Please enter a query and select a model"); + expect(mockSetIsTesting).not.toHaveBeenCalled(); + }); + + it("should set isTesting to true then false around the API call", async () => { + vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({ + data: {}, + headers: { filter: "5->2", tools: "tool-a,tool-b" }, + }); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetIsTesting).toHaveBeenNthCalledWith(1, true); + expect(mockSetIsTesting).toHaveBeenNthCalledWith(2, false); + }); + + it("should clear the previous test result before making a new request", async () => { + vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({ + data: {}, + headers: { filter: "5->2", tools: "tool-a,tool-b" }, + }); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetTestResult).toHaveBeenNthCalledWith(1, null); + }); + + it("should set test result with parsed data on success", async () => { + vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({ + data: {}, + headers: { filter: "10->3", tools: "wiki,github,slack" }, + }); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetTestResult).toHaveBeenCalledWith({ + totalTools: 10, + selectedTools: 3, + tools: ["wiki", "github", "slack"], + }); + expect(NotificationManager.success).toHaveBeenCalledWith( + "Semantic filter test completed successfully" + ); + }); + + it("should show a warning when the filter header is missing", async () => { + vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({ + data: {}, + headers: { filter: null, tools: null }, + }); + + await runSemanticFilterTest(baseArgs); + + expect(NotificationManager.warning).toHaveBeenCalledWith( + "Semantic filter is not enabled or no tools were filtered" + ); + expect(mockSetTestResult).not.toHaveBeenCalledWith(expect.objectContaining({ totalTools: expect.any(Number) })); + }); + + it("should show an error notification and finish testing when the API call fails", async () => { + vi.mocked(testMCPSemanticFilter).mockRejectedValueOnce(new Error("Network error")); + + await runSemanticFilterTest(baseArgs); + + expect(NotificationManager.error).toHaveBeenCalledWith("Failed to test semantic filter"); + expect(mockSetIsTesting).toHaveBeenLastCalledWith(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx index 935d26099c..ae93b11879 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -9,36 +9,6 @@ import NotificationsManager from "./molecules/notifications_manager"; vi.mock("./networking"); -vi.mock("@tremor/react", async (importOriginal) => { - const actual = await importOriginal(); - const React = await import("react"); - const Card = ({ children }: { children: React.ReactNode }) => React.createElement("div", { "data-testid": "card" }, children); - Card.displayName = "Card"; - const Title = ({ children }: { children: React.ReactNode }) => React.createElement("h2", {}, children); - Title.displayName = "Title"; - const Text = ({ children }: { children: React.ReactNode }) => React.createElement("span", {}, children); - Text.displayName = "Text"; - const Divider = () => React.createElement("hr", {}); - Divider.displayName = "Divider"; - const TextInput = ({ value, onChange, placeholder, className }: any) => - React.createElement("input", { - type: "text", - value: value || "", - onChange, - placeholder, - className, - }); - TextInput.displayName = "TextInput"; - return { - ...actual, - Card, - Title, - Text, - Divider, - TextInput, - }; -}); - vi.mock("./common_components/budget_duration_dropdown", () => { const BudgetDurationDropdown = ({ value, onChange }: { value: string | null; onChange: (value: string) => void }) => (