tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-17 02:17:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #25126 from BerriAI/litellm_ui_e2e_psql_pr

Browse Source 
[Test] UI - E2E: Add Playwright tests with local PostgreSQL
This commit is contained in:
yuneng-jiang
2026-04-07 11:59:11 -07:00
committed by GitHub
parent 730ba0f670 184050e2a1
commit 23e1a7d7c2
19 changed files with 726 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ui_e2e_tests/constants.ts
+40
View File
@@ -0,0 +1,40 @@
export const ADMIN_STORAGE_PATH = "admin.storageState.json";
// Page enum — maps to ?page= query parameter values in the UI
export enum Page {
ApiKeys = "api-keys",
Teams = "teams",
AdminSettings = "settings",
}
// Test user credentials — all users have password "test" (hashed in seed.sql)
export enum Role {
ProxyAdmin = "proxy_admin",
ProxyAdminViewer = "proxy_admin_viewer",
InternalUser = "internal_user",
InternalUserViewer = "internal_user_viewer",
TeamAdmin = "team_admin",
}
export const users: Record<Role, { email: string; password: string }> = {
[Role.ProxyAdmin]: {
email: "admin",
password: process.env.LITELLM_MASTER_KEY || "sk-1234",
},
[Role.ProxyAdminViewer]: {
email: "adminviewer@test.local",
password: "test",
},
[Role.InternalUser]: {
email: "internal@test.local",
password: "test",
},
[Role.InternalUserViewer]: {
email: "viewer@test.local",
password: "test",
},
[Role.TeamAdmin]: {
email: "teamadmin@test.local",
password: "test",
},
};
tests/ui_e2e_tests/fixtures/config.yml
+16
View File
@@ -0,0 +1,16 @@
model_list:
- model_name: fake-openai-gpt-4
litellm_params:
model: openai/fake-gpt-4
api_base: os.environ/MOCK_LLM_URL
api_key: fake-key
- model_name: fake-anthropic-claude
litellm_params:
model: openai/fake-claude
api_base: os.environ/MOCK_LLM_URL
api_key: fake-key
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
database_url: os.environ/DATABASE_URL
store_prompts_in_spend_logs: true
tests/ui_e2e_tests/fixtures/mock_llm_server/server.py
+118
View File
@@ -0,0 +1,118 @@
"""
Mock LLM server for UI e2e tests.
Responds to OpenAI-format endpoints with canned responses.
"""
import time
import json
import uuid
import uvicorn
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
app = FastAPI(title="Mock LLM Server")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/health")
async def health():
return {"status": "ok"}
@app.get("/v1/models")
@app.get("/models")
async def list_models():
return {
"object": "list",
"data": [
{"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
{"id": "fake-claude", "object": "model", "owned_by": "mock"},
],
}
@app.post("/v1/chat/completions")
@app.post("/chat/completions")
async def chat_completions(request: Request):
body = await request.json()
model = body.get("model", "mock-model")
stream = body.get("stream", False)
response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
created = int(time.time())
if stream:
async def stream_generator():
chunk = {
"id": response_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": "This is a mock response."},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
done_chunk = {
"id": response_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
}
yield f"data: {json.dumps(done_chunk)}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(
stream_generator(), media_type="text/event-stream"
)
return {
"id": response_id,
"object": "chat.completion",
"created": created,
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "This is a mock response."},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
}
@app.post("/v1/embeddings")
@app.post("/embeddings")
async def embeddings(request: Request):
body = await request.json()
inputs = body.get("input", [""])
if isinstance(inputs, str):
inputs = [inputs]
return {
"object": "list",
"data": [
{"object": "embedding", "index": i, "embedding": [0.0] * 1536}
for i in range(len(inputs))
],
"model": body.get("model", "mock-embedding"),
"usage": {"prompt_tokens": 5, "total_tokens": 5},
}
if __name__ == "__main__":
uvicorn.run(app, host="127.0.0.1", port=8090)
tests/ui_e2e_tests/fixtures/seed.sql
+103
View File
@@ -0,0 +1,103 @@
-- UI E2E Test Database Seed
-- Run with: psql $DATABASE_URL -f seed.sql
-- ============================================================
-- 1. Budget Table (must be first — referenced by org FK)
-- ============================================================
INSERT INTO "LiteLLM_BudgetTable" (
budget_id, max_budget, created_by, updated_by
) VALUES (
'e2e-budget-org', 1000.0, 'e2e-proxy-admin', 'e2e-proxy-admin'
) ON CONFLICT (budget_id) DO NOTHING;
-- ============================================================
-- 2. Organization
-- ============================================================
INSERT INTO "LiteLLM_OrganizationTable" (
organization_id, organization_alias, budget_id, metadata, models, spend,
model_spend, created_by, updated_by
) VALUES (
'e2e-org-main', 'E2E Organization', 'e2e-budget-org', '{}'::jsonb,
ARRAY[]::text[], 0.0, '{}'::jsonb, 'e2e-proxy-admin', 'e2e-proxy-admin'
) ON CONFLICT (organization_id) DO NOTHING;
-- ============================================================
-- 3. Users (password is scrypt hash of "test")
-- ============================================================
INSERT INTO "LiteLLM_UserTable" (
user_id, user_email, user_role, password, teams, models, metadata,
spend, model_spend, model_max_budget
) VALUES
(
'e2e-proxy-admin', 'admin@test.local', 'proxy_admin', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
ARRAY['e2e-team-crud']::text[], ARRAY[]::text[], '{}'::jsonb,
0.0, '{}'::jsonb, '{}'::jsonb
),
(
'e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
0.0, '{}'::jsonb, '{}'::jsonb
),
(
'e2e-internal-user', 'internal@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
ARRAY['e2e-team-crud', 'e2e-team-org']::text[], ARRAY[]::text[], '{}'::jsonb,
0.0, '{}'::jsonb, '{}'::jsonb
),
(
'e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
0.0, '{}'::jsonb, '{}'::jsonb
),
(
'e2e-team-admin', 'teamadmin@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
ARRAY['e2e-team-crud', 'e2e-team-delete']::text[], ARRAY[]::text[], '{}'::jsonb,
0.0, '{}'::jsonb, '{}'::jsonb
)
ON CONFLICT (user_id) DO NOTHING;
-- ============================================================
-- 4. Teams
-- ============================================================
INSERT INTO "LiteLLM_TeamTable" (
team_id, team_alias, organization_id, admins, members,
members_with_roles, metadata, models, spend, model_spend,
model_max_budget, blocked
) VALUES
(
'e2e-team-crud', 'E2E Team CRUD', NULL,
ARRAY['e2e-team-admin']::text[],
ARRAY['e2e-team-admin', 'e2e-internal-user']::text[],
'[{"role": "admin", "user_id": "e2e-team-admin"}, {"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
'{}'::jsonb,
ARRAY['fake-openai-gpt-4', 'fake-anthropic-claude']::text[],
0.0, '{}'::jsonb, '{}'::jsonb, false
),
(
'e2e-team-delete', 'E2E Team Delete', NULL,
ARRAY['e2e-team-admin']::text[],
ARRAY['e2e-team-admin']::text[],
'[{"role": "admin", "user_id": "e2e-team-admin"}]'::jsonb,
'{}'::jsonb,
ARRAY['fake-openai-gpt-4']::text[],
0.0, '{}'::jsonb, '{}'::jsonb, false
),
(
'e2e-team-org', 'E2E Team In Org', 'e2e-org-main',
ARRAY[]::text[],
ARRAY['e2e-internal-user']::text[],
'[{"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
'{}'::jsonb,
ARRAY['fake-openai-gpt-4']::text[],
0.0, '{}'::jsonb, '{}'::jsonb, false
)
ON CONFLICT (team_id) DO NOTHING;
-- ============================================================
-- 5. Team Memberships
-- ============================================================
INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend) VALUES
('e2e-team-admin', 'e2e-team-crud', 0.0),
('e2e-internal-user', 'e2e-team-crud', 0.0),
('e2e-team-admin', 'e2e-team-delete', 0.0),
('e2e-internal-user', 'e2e-team-org', 0.0)
ON CONFLICT (user_id, team_id) DO NOTHING;
tests/ui_e2e_tests/globalSetup.ts
+32
View File
@@ -0,0 +1,32 @@
import { chromium, expect } from "@playwright/test";
import { users, Role, ADMIN_STORAGE_PATH } from "./constants";
import * as fs from "fs";
async function globalSetup() {
const browser = await chromium.launch();
const page = await browser.newPage();
await page.goto("http://localhost:4000/ui/login");
await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
await page.getByRole("button", { name: "Login", exact: true }).click();
try {
// Wait for navigation away from login page into the dashboard
await page.waitForURL(
(url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"),
{ timeout: 30_000 },
);
// Wait for sidebar to render as a signal that the dashboard is ready
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
} catch (e) {
// Save a screenshot for debugging before re-throwing
fs.mkdirSync("test-results", { recursive: true });
await page.screenshot({ path: "test-results/global-setup-failure.png", fullPage: true });
console.error("Global setup failed. Screenshot saved to test-results/global-setup-failure.png");
console.error("Current URL:", page.url());
throw e;
}
await page.context().storageState({ path: ADMIN_STORAGE_PATH });
await browser.close();
}
export default globalSetup;
tests/ui_e2e_tests/helpers/login.ts
+16
View File
@@ -0,0 +1,16 @@
import { Page as PlaywrightPage, expect } from "@playwright/test";
import { users, Role } from "../constants";
export async function loginAs(page: PlaywrightPage, role: Role) {
const user = users[role];
await page.goto("/ui/login");
await page.getByPlaceholder("Enter your username").fill(user.email);
await page.getByPlaceholder("Enter your password").fill(user.password);
await page.getByRole("button", { name: "Login", exact: true }).click();
// Wait for navigation away from login page into the dashboard
await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), {
timeout: 30_000,
});
// Wait for sidebar to render as a signal that the dashboard is ready
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
}
tests/ui_e2e_tests/helpers/navigation.ts
+6
View File
@@ -0,0 +1,6 @@
import { Page as PlaywrightPage } from "@playwright/test";
import { Page } from "../constants";
export async function navigateToPage(page: PlaywrightPage, targetPage: Page) {
await page.goto(`/ui?page=${targetPage}`);
}
tests/ui_e2e_tests/package-lock.json
Generated
+76
View File
@@ -0,0 +1,76 @@
{
"name": "litellm-ui-e2e-tests",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "litellm-ui-e2e-tests",
"devDependencies": {
"@playwright/test": "^1.50.0"
}
},
"node_modules/@playwright/test": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz",
"integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright": "1.59.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/playwright": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.59.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=18"
}
}
}
}
tests/ui_e2e_tests/package.json
+12
View File
@@ -0,0 +1,12 @@
{
"name": "litellm-ui-e2e-tests",
"private": true,
"devDependencies": {
"@playwright/test": "^1.50.0"
},
"scripts": {
"e2e": "playwright test",
"e2e:headed": "playwright test --headed",
"e2e:ui": "playwright test --ui"
}
}
tests/ui_e2e_tests/playwright.config.ts
+31
View File
@@ -0,0 +1,31 @@
import { defineConfig, devices } from "@playwright/test";
const isCI = !!process.env.CI;
export default defineConfig({
testDir: "./tests",
testMatch: "**/*.spec.ts",
globalSetup: "./globalSetup.ts",
fullyParallel: false,
forbidOnly: isCI,
retries: isCI ? 2 : 0,
workers: 1,
reporter: isCI ? [["html", { open: "never" }]] : [["html"]],
timeout: 4 * 60 * 1000,
expect: {
timeout: 10_000,
},
use: {
baseURL: "http://localhost:4000",
trace: "on-first-retry",
screenshot: "only-on-failure",
actionTimeout: 15_000,
navigationTimeout: 30_000,
},
projects: [
{
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
],
});
tests/ui_e2e_tests/run_e2e.sh
Executable
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env bash
set -euo pipefail
# ================================================================
# UI E2E Test Runner
# Starts postgres, seeds DB, starts mock + proxy, runs Playwright.
# All credentials are generated per run — nothing is stored on disk.
#
# In CI (CI=true), expects:
# - PostgreSQL already running on 127.0.0.1:5432
# - DATABASE_URL already set
# - Python/Poetry already installed
# - Node.js/npx already available
# ================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
IS_CI="${CI:-false}"
CONTAINER_NAME="litellm-e2e-postgres-$$"
MOCK_PID=""
PROXY_PID=""
# --- Ensure common tool paths are available (local dev only) ---
if [ "$IS_CI" = "false" ]; then
for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do
[ -d "$p" ] && export PATH="$p:$PATH"
done
[ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
fi
# --- Cleanup on exit ---
cleanup() {
echo "Cleaning up..."
[ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true
[ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true
if [ "$IS_CI" = "false" ]; then
docker stop "$CONTAINER_NAME" 2>/dev/null || true
fi
echo "Done."
}
trap cleanup EXIT INT TERM
# --- Pre-flight checks ---
for cmd in python3 npx poetry; do
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
done
# --- Database setup ---
if [ "$IS_CI" = "false" ]; then
# Local: spin up a postgres container
for cmd in docker psql; do
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
done
for port in 4000 5432 8090; do
if lsof -ti ":$port" >/dev/null 2>&1; then
echo "Error: port $port is in use"
exit 1
fi
done
export POSTGRES_USER="e2euser"
export POSTGRES_PASSWORD="$(openssl rand -hex 32)"
export POSTGRES_DB="litellm_e2e"
export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}"
echo "=== Starting PostgreSQL ==="
docker run -d --rm --name "$CONTAINER_NAME" \
-e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \
-p 127.0.0.1:5432:5432 \
postgres:16
echo "Waiting for PostgreSQL..."
for i in $(seq 1 30); do
if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
break
fi
sleep 1
done
else
# CI: postgres is already running as a service container
echo "=== Using CI PostgreSQL service ==="
: "${DATABASE_URL:?DATABASE_URL must be set in CI}"
fi
# --- Credentials ---
export LITELLM_MASTER_KEY="sk-e2e-$(openssl rand -hex 32)"
export MOCK_LLM_URL="http://127.0.0.1:8090/v1"
export DISABLE_SCHEMA_UPDATE="true"
# --- Python environment ---
echo "=== Setting up Python environment ==="
cd "$REPO_ROOT"
if ! poetry run python3 -c "import prisma" 2>/dev/null; then
echo "Installing Python dependencies (first run)..."
poetry install --with dev,proxy-dev --extras "proxy" --quiet
poetry run pip install nodejs-wheel-binaries 2>/dev/null || true
poetry run prisma generate --schema litellm/proxy/schema.prisma
fi
echo "=== Pushing Prisma schema to database ==="
poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
# --- Mock LLM server ---
echo "=== Starting mock LLM server ==="
poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" &
MOCK_PID=$!
for i in $(seq 1 15); do
if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi
sleep 1
done
# --- LiteLLM proxy ---
echo "=== Starting LiteLLM proxy ==="
cd "$REPO_ROOT"
poetry run python3 -m litellm.proxy.proxy_cli \
--config "$SCRIPT_DIR/fixtures/config.yml" \
--port 4000 &
PROXY_PID=$!
echo "Waiting for proxy..."
PROXY_READY=0
for i in $(seq 1 180); do
if ! kill -0 "$PROXY_PID" 2>/dev/null; then
echo "Error: proxy process exited unexpectedly"
exit 1
fi
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
PROXY_READY=1
break
fi
sleep 1
done
if [ "$PROXY_READY" -ne 1 ]; then
echo "Error: proxy did not become healthy within 180 seconds"
exit 1
fi
echo "Proxy is ready."
# --- Seed database ---
echo "=== Seeding database ==="
# Extract credentials from DATABASE_URL for psql
DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p')
DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p')
DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p')
DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p')
PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-f "$SCRIPT_DIR/fixtures/seed.sql"
# --- Playwright ---
echo "=== Installing Playwright dependencies ==="
cd "$SCRIPT_DIR"
npm install --silent
echo "=== Running Playwright tests ==="
npx playwright test "$@"
EXIT_CODE=$?
exit $EXIT_CODE
tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts
+10
View File
@@ -0,0 +1,10 @@
import { test, expect } from "@playwright/test";
import { Role } from "../../constants";
import { loginAs } from "../../helpers/login";
test.describe("Admin Viewer Role", () => {
test("Should not see Test Key page", async ({ page }) => {
await loginAs(page, Role.ProxyAdminViewer);
await expect(page.getByRole("menuitem", { name: "Test Key" })).not.toBeVisible();
});
});
tests/ui_e2e_tests/tests/roles/internal-user.spec.ts
+12
View File
@@ -0,0 +1,12 @@
import { test, expect } from "@playwright/test";
import { Page, Role } from "../../constants";
import { loginAs } from "../../helpers/login";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Internal User Role", () => {
test("Should not see litellm-dashboard keys", async ({ page }) => {
await loginAs(page, Role.InternalUser);
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByText("litellm-dashboard")).not.toBeVisible();
});
});
tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts
+28
View File
@@ -0,0 +1,28 @@
import { test, expect } from "@playwright/test";
import { Page, Role } from "../../constants";
import { loginAs } from "../../helpers/login";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Internal User Viewer Role", () => {
test("Can only see allowed pages", async ({ page }) => {
await loginAs(page, Role.InternalUserViewer);
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible();
await expect(page.getByRole("menuitem", { name: "Admin Settings" })).not.toBeVisible();
});
test("Cannot create keys", async ({ page }) => {
await loginAs(page, Role.InternalUserViewer);
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: /Create New Key/i })).not.toBeVisible();
});
test("Cannot edit or delete keys", async ({ page }) => {
await loginAs(page, Role.InternalUserViewer);
await navigateToPage(page, Page.ApiKeys);
// Ensure the keys table has loaded before asserting absence of actions
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible();
await expect(page.getByRole("button", { name: /Edit Key/i })).not.toBeVisible();
await expect(page.getByRole("button", { name: /Delete Key/i })).not.toBeVisible();
await expect(page.getByRole("button", { name: /Regenerate Key/i })).not.toBeVisible();
});
});
tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts
+21
View File
@@ -0,0 +1,21 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH, Page, Role, users } from "../../constants";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Proxy Admin Role", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Can create keys", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: /Create New Key/i })).toBeVisible();
});
test("Can list teams via API", async ({ page }) => {
const response = await page.request.get("/team/list", {
headers: {
Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
},
});
expect(response.status()).toBe(200);
});
});
tests/ui_e2e_tests/tests/roles/team-admin.spec.ts
+13
View File
@@ -0,0 +1,13 @@
import { test, expect } from "@playwright/test";
import { Page, Role } from "../../constants";
import { loginAs } from "../../helpers/login";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Team Admin Role", () => {
test("Can view team keys but not admin settings", async ({ page }) => {
await loginAs(page, Role.TeamAdmin);
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible();
await expect(page.getByRole("menuitem", { name: "Admin Settings" })).not.toBeVisible();
});
});
tests/ui_e2e_tests/tests/security/login-logout.spec.ts
+18
View File
@@ -0,0 +1,18 @@
import { test, expect } from "@playwright/test";
import { users, Role } from "../../constants";
test.describe("Authentication", () => {
test("Login with valid admin credentials", async ({ page }) => {
await page.goto("/ui/login");
await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
await page.getByRole("button", { name: "Login", exact: true }).click();
await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible();
});
test("Unauthenticated user is redirected to login", async ({ page }) => {
await page.goto("/ui");
await page.waitForURL(/\/ui\/login/);
await expect(page.getByRole("heading", { name: /Login/i })).toBeVisible();
});
});
tests/ui_e2e_tests/tsconfig.json
+11
View File
@@ -0,0 +1,11 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"strict": true,
"esModuleInterop": true,
"outDir": "./dist",
"rootDir": "."
},
"include": ["**/*.ts"]
}
ui/litellm-dashboard/package.json
+1
View File
@@ -16,6 +16,7 @@
"format:check": "prettier --check .",
"e2e": "playwright test --config e2e_tests/playwright.config.ts",
"e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts",
"e2e:psql": "../../tests/ui_e2e_tests/run_e2e.sh",
"knip": "knip",
"knip:fix": "knip --fix"
},