diff --git a/.github/workflows/test-litellm-ui-e2e.yml b/.github/workflows/test-litellm-ui-e2e.yml new file mode 100644 index 0000000000..369e749cda --- /dev/null +++ b/.github/workflows/test-litellm-ui-e2e.yml @@ -0,0 +1,99 @@ +name: UI E2E Playwright Tests + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ui_e2e_tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-ui-e2e-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-ui-e2e- + ${{ runner.os }}-poetry- + + - name: Install Python dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy" --quiet + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Install Playwright + run: | + cd tests/ui_e2e_tests + npm install --silent + npx playwright install --with-deps chromium + + - name: Run UI E2E tests + env: + CI: "true" + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + run: | + cd tests/ui_e2e_tests + ./run_e2e.sh + + - name: Upload test results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: playwright-results + path: | + tests/ui_e2e_tests/test-results/ + tests/ui_e2e_tests/playwright-report/ + retention-days: 7 diff --git a/tests/ui_e2e_tests/constants.ts b/tests/ui_e2e_tests/constants.ts new file mode 100644 index 0000000000..484cde30fc --- /dev/null +++ b/tests/ui_e2e_tests/constants.ts @@ -0,0 +1,40 @@ +export const ADMIN_STORAGE_PATH = "admin.storageState.json"; + +// Page enum — maps to ?page= query parameter values in the UI +export enum Page { + ApiKeys = "api-keys", + Teams = "teams", + AdminSettings = "settings", +} + +// Test user credentials — all users have password "test" (hashed in seed.sql) +export enum Role { + ProxyAdmin = "proxy_admin", + ProxyAdminViewer = "proxy_admin_viewer", + InternalUser = "internal_user", + InternalUserViewer = "internal_user_viewer", + TeamAdmin = "team_admin", +} + +export const users: Record = { + [Role.ProxyAdmin]: { + email: "admin", + password: process.env.LITELLM_MASTER_KEY || "sk-1234", + }, + [Role.ProxyAdminViewer]: { + email: "adminviewer@test.local", + password: "test", + }, + [Role.InternalUser]: { + email: "internal@test.local", + password: "test", + }, + [Role.InternalUserViewer]: { + email: "viewer@test.local", + password: "test", + }, + [Role.TeamAdmin]: { + email: "teamadmin@test.local", + password: "test", + }, +}; diff --git a/tests/ui_e2e_tests/fixtures/config.yml b/tests/ui_e2e_tests/fixtures/config.yml new file mode 100644 index 0000000000..438c236b03 --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/config.yml @@ -0,0 +1,16 @@ +model_list: + - model_name: fake-openai-gpt-4 + litellm_params: + model: openai/fake-gpt-4 + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + - model_name: fake-anthropic-claude + litellm_params: + model: openai/fake-claude + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_prompts_in_spend_logs: true diff --git a/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py b/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py new file mode 100644 index 0000000000..7a0699016a --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py @@ -0,0 +1,118 @@ +""" +Mock LLM server for UI e2e tests. +Responds to OpenAI-format endpoints with canned responses. +""" + +import time +import json +import uuid + +import uvicorn +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse + + +app = FastAPI(title="Mock LLM Server") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/v1/models") +@app.get("/models") +async def list_models(): + return { + "object": "list", + "data": [ + {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"}, + {"id": "fake-claude", "object": "model", "owned_by": "mock"}, + ], + } + + +@app.post("/v1/chat/completions") +@app.post("/chat/completions") +async def chat_completions(request: Request): + body = await request.json() + model = body.get("model", "mock-model") + stream = body.get("stream", False) + + response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + if stream: + async def stream_generator(): + chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "This is a mock response."}, + "finish_reason": None, + } + ], + } + yield f"data: {json.dumps(chunk)}\n\n" + + done_chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + yield f"data: {json.dumps(done_chunk)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse( + stream_generator(), media_type="text/event-stream" + ) + + return { + "id": response_id, + "object": "chat.completion", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "This is a mock response."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, + } + + +@app.post("/v1/embeddings") +@app.post("/embeddings") +async def embeddings(request: Request): + body = await request.json() + inputs = body.get("input", [""]) + if isinstance(inputs, str): + inputs = [inputs] + return { + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": [0.0] * 1536} + for i in range(len(inputs)) + ], + "model": body.get("model", "mock-embedding"), + "usage": {"prompt_tokens": 5, "total_tokens": 5}, + } + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8090) diff --git a/tests/ui_e2e_tests/fixtures/seed.sql b/tests/ui_e2e_tests/fixtures/seed.sql new file mode 100644 index 0000000000..f271a74817 --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/seed.sql @@ -0,0 +1,103 @@ +-- UI E2E Test Database Seed +-- Run with: psql $DATABASE_URL -f seed.sql + +-- ============================================================ +-- 1. Budget Table (must be first — referenced by org FK) +-- ============================================================ +INSERT INTO "LiteLLM_BudgetTable" ( + budget_id, max_budget, created_by, updated_by +) VALUES ( + 'e2e-budget-org', 1000.0, 'e2e-proxy-admin', 'e2e-proxy-admin' +) ON CONFLICT (budget_id) DO NOTHING; + +-- ============================================================ +-- 2. Organization +-- ============================================================ +INSERT INTO "LiteLLM_OrganizationTable" ( + organization_id, organization_alias, budget_id, metadata, models, spend, + model_spend, created_by, updated_by +) VALUES ( + 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', '{}'::jsonb, + ARRAY[]::text[], 0.0, '{}'::jsonb, 'e2e-proxy-admin', 'e2e-proxy-admin' +) ON CONFLICT (organization_id) DO NOTHING; + +-- ============================================================ +-- 3. Users (password is scrypt hash of "test") +-- ============================================================ +INSERT INTO "LiteLLM_UserTable" ( + user_id, user_email, user_role, password, teams, models, metadata, + spend, model_spend, model_max_budget +) VALUES +( + 'e2e-proxy-admin', 'admin@test.local', 'proxy_admin', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-internal-user', 'internal@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud', 'e2e-team-org']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-team-admin', 'teamadmin@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud', 'e2e-team-delete']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +) +ON CONFLICT (user_id) DO NOTHING; + +-- ============================================================ +-- 4. Teams +-- ============================================================ +INSERT INTO "LiteLLM_TeamTable" ( + team_id, team_alias, organization_id, admins, members, + members_with_roles, metadata, models, spend, model_spend, + model_max_budget, blocked +) VALUES +( + 'e2e-team-crud', 'E2E Team CRUD', NULL, + ARRAY['e2e-team-admin']::text[], + ARRAY['e2e-team-admin', 'e2e-internal-user']::text[], + '[{"role": "admin", "user_id": "e2e-team-admin"}, {"role": "user", "user_id": "e2e-internal-user"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4', 'fake-anthropic-claude']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +), +( + 'e2e-team-delete', 'E2E Team Delete', NULL, + ARRAY['e2e-team-admin']::text[], + ARRAY['e2e-team-admin']::text[], + '[{"role": "admin", "user_id": "e2e-team-admin"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +), +( + 'e2e-team-org', 'E2E Team In Org', 'e2e-org-main', + ARRAY[]::text[], + ARRAY['e2e-internal-user']::text[], + '[{"role": "user", "user_id": "e2e-internal-user"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +) +ON CONFLICT (team_id) DO NOTHING; + +-- ============================================================ +-- 5. Team Memberships +-- ============================================================ +INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend) VALUES + ('e2e-team-admin', 'e2e-team-crud', 0.0), + ('e2e-internal-user', 'e2e-team-crud', 0.0), + ('e2e-team-admin', 'e2e-team-delete', 0.0), + ('e2e-internal-user', 'e2e-team-org', 0.0) +ON CONFLICT (user_id, team_id) DO NOTHING; diff --git a/tests/ui_e2e_tests/globalSetup.ts b/tests/ui_e2e_tests/globalSetup.ts new file mode 100644 index 0000000000..ff82cded2c --- /dev/null +++ b/tests/ui_e2e_tests/globalSetup.ts @@ -0,0 +1,32 @@ +import { chromium, expect } from "@playwright/test"; +import { users, Role, ADMIN_STORAGE_PATH } from "./constants"; +import * as fs from "fs"; + +async function globalSetup() { + const browser = await chromium.launch(); + const page = await browser.newPage(); + await page.goto("http://localhost:4000/ui/login"); + await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); + await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + try { + // Wait for navigation away from login page into the dashboard + await page.waitForURL( + (url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), + { timeout: 30_000 }, + ); + // Wait for sidebar to render as a signal that the dashboard is ready + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); + } catch (e) { + // Save a screenshot for debugging before re-throwing + fs.mkdirSync("test-results", { recursive: true }); + await page.screenshot({ path: "test-results/global-setup-failure.png", fullPage: true }); + console.error("Global setup failed. Screenshot saved to test-results/global-setup-failure.png"); + console.error("Current URL:", page.url()); + throw e; + } + await page.context().storageState({ path: ADMIN_STORAGE_PATH }); + await browser.close(); +} + +export default globalSetup; diff --git a/tests/ui_e2e_tests/helpers/login.ts b/tests/ui_e2e_tests/helpers/login.ts new file mode 100644 index 0000000000..d1c1ac410f --- /dev/null +++ b/tests/ui_e2e_tests/helpers/login.ts @@ -0,0 +1,16 @@ +import { Page as PlaywrightPage, expect } from "@playwright/test"; +import { users, Role } from "../constants"; + +export async function loginAs(page: PlaywrightPage, role: Role) { + const user = users[role]; + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(user.email); + await page.getByPlaceholder("Enter your password").fill(user.password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + // Wait for navigation away from login page into the dashboard + await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), { + timeout: 30_000, + }); + // Wait for sidebar to render as a signal that the dashboard is ready + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); +} diff --git a/tests/ui_e2e_tests/helpers/navigation.ts b/tests/ui_e2e_tests/helpers/navigation.ts new file mode 100644 index 0000000000..065e218053 --- /dev/null +++ b/tests/ui_e2e_tests/helpers/navigation.ts @@ -0,0 +1,6 @@ +import { Page as PlaywrightPage } from "@playwright/test"; +import { Page } from "../constants"; + +export async function navigateToPage(page: PlaywrightPage, targetPage: Page) { + await page.goto(`/ui?page=${targetPage}`); +} diff --git a/tests/ui_e2e_tests/package-lock.json b/tests/ui_e2e_tests/package-lock.json new file mode 100644 index 0000000000..5ce8059b78 --- /dev/null +++ b/tests/ui_e2e_tests/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "litellm-ui-e2e-tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "litellm-ui-e2e-tests", + "devDependencies": { + "@playwright/test": "^1.50.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/tests/ui_e2e_tests/package.json b/tests/ui_e2e_tests/package.json new file mode 100644 index 0000000000..b57fcd51e5 --- /dev/null +++ b/tests/ui_e2e_tests/package.json @@ -0,0 +1,12 @@ +{ + "name": "litellm-ui-e2e-tests", + "private": true, + "devDependencies": { + "@playwright/test": "^1.50.0" + }, + "scripts": { + "e2e": "playwright test", + "e2e:headed": "playwright test --headed", + "e2e:ui": "playwright test --ui" + } +} diff --git a/tests/ui_e2e_tests/playwright.config.ts b/tests/ui_e2e_tests/playwright.config.ts new file mode 100644 index 0000000000..6c6fd2d5e9 --- /dev/null +++ b/tests/ui_e2e_tests/playwright.config.ts @@ -0,0 +1,31 @@ +import { defineConfig, devices } from "@playwright/test"; + +const isCI = !!process.env.CI; + +export default defineConfig({ + testDir: "./tests", + testMatch: "**/*.spec.ts", + globalSetup: "./globalSetup.ts", + fullyParallel: false, + forbidOnly: isCI, + retries: isCI ? 2 : 0, + workers: 1, + reporter: isCI ? [["html", { open: "never" }]] : [["html"]], + timeout: 4 * 60 * 1000, + expect: { + timeout: 10_000, + }, + use: { + baseURL: "http://localhost:4000", + trace: "on-first-retry", + screenshot: "only-on-failure", + actionTimeout: 15_000, + navigationTimeout: 30_000, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], +}); diff --git a/tests/ui_e2e_tests/run_e2e.sh b/tests/ui_e2e_tests/run_e2e.sh new file mode 100755 index 0000000000..400e53258d --- /dev/null +++ b/tests/ui_e2e_tests/run_e2e.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ================================================================ +# UI E2E Test Runner +# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. +# All credentials are generated per run — nothing is stored on disk. +# +# In CI (CI=true), expects: +# - PostgreSQL already running on 127.0.0.1:5432 +# - DATABASE_URL already set +# - Python/Poetry already installed +# - Node.js/npx already available +# ================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +IS_CI="${CI:-false}" +CONTAINER_NAME="litellm-e2e-postgres-$$" +MOCK_PID="" +PROXY_PID="" + +# --- Ensure common tool paths are available (local dev only) --- +if [ "$IS_CI" = "false" ]; then + for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do + [ -d "$p" ] && export PATH="$p:$PATH" + done + [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" +fi + +# --- Cleanup on exit --- +cleanup() { + echo "Cleaning up..." + [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true + [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true + if [ "$IS_CI" = "false" ]; then + docker stop "$CONTAINER_NAME" 2>/dev/null || true + fi + echo "Done." +} +trap cleanup EXIT INT TERM + +# --- Pre-flight checks --- +for cmd in python3 npx poetry; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } +done + +# --- Database setup --- +if [ "$IS_CI" = "false" ]; then + # Local: spin up a postgres container + for cmd in docker psql; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } + done + for port in 4000 5432 8090; do + if lsof -ti ":$port" >/dev/null 2>&1; then + echo "Error: port $port is in use" + exit 1 + fi + done + + export POSTGRES_USER="e2euser" + export POSTGRES_PASSWORD="$(openssl rand -hex 32)" + export POSTGRES_DB="litellm_e2e" + export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}" + + echo "=== Starting PostgreSQL ===" + docker run -d --rm --name "$CONTAINER_NAME" \ + -e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \ + -p 127.0.0.1:5432:5432 \ + postgres:16 + + echo "Waiting for PostgreSQL..." + for i in $(seq 1 30); do + if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + break + fi + sleep 1 + done +else + # CI: postgres is already running as a service container + echo "=== Using CI PostgreSQL service ===" + : "${DATABASE_URL:?DATABASE_URL must be set in CI}" +fi + +# --- Credentials --- +export LITELLM_MASTER_KEY="sk-e2e-$(openssl rand -hex 32)" +export MOCK_LLM_URL="http://127.0.0.1:8090/v1" +export DISABLE_SCHEMA_UPDATE="true" + +# --- Python environment --- +echo "=== Setting up Python environment ===" +cd "$REPO_ROOT" +if ! poetry run python3 -c "import prisma" 2>/dev/null; then + echo "Installing Python dependencies (first run)..." + poetry install --with dev,proxy-dev --extras "proxy" --quiet + poetry run pip install nodejs-wheel-binaries 2>/dev/null || true + poetry run prisma generate --schema litellm/proxy/schema.prisma +fi + +echo "=== Pushing Prisma schema to database ===" +poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + +# --- Mock LLM server --- +echo "=== Starting mock LLM server ===" +poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" & +MOCK_PID=$! + +for i in $(seq 1 15); do + if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi + sleep 1 +done + +# --- LiteLLM proxy --- +echo "=== Starting LiteLLM proxy ===" +cd "$REPO_ROOT" +poetry run python3 -m litellm.proxy.proxy_cli \ + --config "$SCRIPT_DIR/fixtures/config.yml" \ + --port 4000 & +PROXY_PID=$! + +echo "Waiting for proxy..." +PROXY_READY=0 +for i in $(seq 1 180); do + if ! kill -0 "$PROXY_PID" 2>/dev/null; then + echo "Error: proxy process exited unexpectedly" + exit 1 + fi + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + PROXY_READY=1 + break + fi + sleep 1 +done +if [ "$PROXY_READY" -ne 1 ]; then + echo "Error: proxy did not become healthy within 180 seconds" + exit 1 +fi +echo "Proxy is ready." + +# --- Seed database --- +echo "=== Seeding database ===" +# Extract credentials from DATABASE_URL for psql +DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') +DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') +DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') +DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') +DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') + +PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ + -f "$SCRIPT_DIR/fixtures/seed.sql" + +# --- Playwright --- +echo "=== Installing Playwright dependencies ===" +cd "$SCRIPT_DIR" +npm install --silent + +echo "=== Running Playwright tests ===" +npx playwright test "$@" +EXIT_CODE=$? + +exit $EXIT_CODE diff --git a/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts b/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts new file mode 100644 index 0000000000..6e29d7c22e --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts @@ -0,0 +1,10 @@ +import { test, expect } from "@playwright/test"; +import { Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; + +test.describe("Admin Viewer Role", () => { + test("Should not see Test Key page", async ({ page }) => { + await loginAs(page, Role.ProxyAdminViewer); + await expect(page.getByRole("menuitem", { name: "Test Key" })).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts b/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts new file mode 100644 index 0000000000..4cb39a1ce7 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts @@ -0,0 +1,12 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Internal User Role", () => { + test("Should not see litellm-dashboard keys", async ({ page }) => { + await loginAs(page, Role.InternalUser); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByText("litellm-dashboard")).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts b/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts new file mode 100644 index 0000000000..3ca17caff1 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts @@ -0,0 +1,28 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Internal User Viewer Role", () => { + test("Can only see allowed pages", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + await expect(page.getByRole("menuitem", { name: "Admin Settings" })).not.toBeVisible(); + }); + + test("Cannot create keys", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: /Create New Key/i })).not.toBeVisible(); + }); + + test("Cannot edit or delete keys", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await navigateToPage(page, Page.ApiKeys); + // Ensure the keys table has loaded before asserting absence of actions + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + await expect(page.getByRole("button", { name: /Edit Key/i })).not.toBeVisible(); + await expect(page.getByRole("button", { name: /Delete Key/i })).not.toBeVisible(); + await expect(page.getByRole("button", { name: /Regenerate Key/i })).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts b/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts new file mode 100644 index 0000000000..bf159175e8 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts @@ -0,0 +1,21 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, Page } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Proxy Admin Role", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Can create keys", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: /Create New Key/i })).toBeVisible(); + }); + + test("Can list teams via API", async ({ page }) => { + const response = await page.request.get("/team/list", { + headers: { + Authorization: `Bearer ${process.env.LITELLM_MASTER_KEY || "sk-1234"}`, + }, + }); + expect(response.status()).toBe(200); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts b/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts new file mode 100644 index 0000000000..b0e93ea1f9 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts @@ -0,0 +1,12 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Team Admin Role", () => { + test("Can view all team keys", async ({ page }) => { + await loginAs(page, Role.TeamAdmin); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/security/login-logout.spec.ts b/tests/ui_e2e_tests/tests/security/login-logout.spec.ts new file mode 100644 index 0000000000..88ce3f1a6d --- /dev/null +++ b/tests/ui_e2e_tests/tests/security/login-logout.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from "@playwright/test"; +import { users, Role } from "../../constants"; + +test.describe("Authentication", () => { + test("Login with valid admin credentials", async ({ page }) => { + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); + await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + }); + + test("Unauthenticated user is redirected to login", async ({ page }) => { + await page.goto("/ui"); + await page.waitForURL(/\/ui\/login/); + await expect(page.getByRole("heading", { name: /Login/i })).toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tsconfig.json b/tests/ui_e2e_tests/tsconfig.json new file mode 100644 index 0000000000..8d92d3764e --- /dev/null +++ b/tests/ui_e2e_tests/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": ["**/*.ts"] +} diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 7b763eaa66..4392fd99d9 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -16,70 +16,70 @@ "format:check": "prettier --check .", "e2e": "playwright test --config e2e_tests/playwright.config.ts", "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts", + "e2e:psql": "../../tests/ui_e2e_tests/run_e2e.sh", "knip": "knip", "knip:fix": "knip --fix" }, "dependencies": { - "@anthropic-ai/sdk": "^0.54.0", - "@headlessui/tailwindcss": "^0.2.0", - "@heroicons/react": "^1.0.6", - "@remixicon/react": "^4.1.1", - "@tanstack/react-pacer": "^0.2.0", - "@tanstack/react-query": "^5.64.1", - "@tanstack/react-table": "^8.20.6", - "@tremor/react": "^3.13.3", - "@types/papaparse": "^5.3.15", - "antd": "^5.13.2", - "cva": "^1.0.0-beta.3", - "dayjs": "^1.11.19", - "jwt-decode": "^4.0.0", - "lucide-react": "^0.513.0", - "moment": "^2.30.1", - "next": "^16.1.7", - "openai": "^4.93.0", - "papaparse": "^5.5.2", - "react": "^18.3.1", - "react-copy-to-clipboard": "^5.1.0", - "react-dom": "^18.3.1", - "react-json-view-lite": "^2.5.0", - "react-markdown": "^9.0.1", - "react-syntax-highlighter": "^15.6.6", - "remark-gfm": "^4.0.1", - "tailwind-merge": "^3.2.0", - "uuid": "^11.1.0" + "@anthropic-ai/sdk": "0.54.0", + "@headlessui/tailwindcss": "0.2.2", + "@heroicons/react": "1.0.6", + "@remixicon/react": "4.9.0", + "@tanstack/react-pacer": "0.2.0", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-table": "8.21.3", + "@tremor/react": "3.18.7", + "@types/papaparse": "5.5.2", + "antd": "5.29.3", + "cva": "1.0.0-beta.4", + "dayjs": "1.11.19", + "jwt-decode": "4.0.0", + "lucide-react": "0.513.0", + "moment": "2.30.1", + "next": "16.1.7", + "openai": "4.104.0", + "papaparse": "5.5.3", + "react": "18.3.1", + "react-copy-to-clipboard": "5.1.0", + "react-dom": "18.3.1", + "react-json-view-lite": "2.5.0", + "react-markdown": "9.1.0", + "react-syntax-highlighter": "15.6.6", + "remark-gfm": "4.0.1", + "tailwind-merge": "3.4.0", + "uuid": "11.1.0" }, "devDependencies": { - "@neondatabase/api-client": "^2.6.0", - "@playwright/test": "^1.57.0", - "@tailwindcss/forms": "^0.5.7", - "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^16.3.0", - "@testing-library/user-event": "^14.6.1", - "@types/babel__traverse": "^7.28.0", - "@types/lodash": "^4.17.15", + "@playwright/test": "1.58.1", + "@tailwindcss/forms": "0.5.11", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/react": "16.3.2", + "@testing-library/user-event": "14.6.1", + "@types/babel__traverse": "7.28.0", + "@types/lodash": "4.17.23", "@types/node": "20.19.37", "@types/react": "18.2.48", - "@types/react-copy-to-clipboard": "^5.0.7", - "@types/react-dom": "^18", - "@types/react-syntax-highlighter": "^15.5.11", - "@types/uuid": "^10.0.0", - "@vitest/coverage-v8": "^3.2.4", - "@vitest/ui": "^3.2.4", - "autoprefixer": "^10.4.17", - "dotenv": "^17.2.3", - "eslint": "^9.39.2", + "@types/react-copy-to-clipboard": "5.0.7", + "@types/react-dom": "18.3.7", + "@types/react-syntax-highlighter": "15.5.13", + "@types/uuid": "10.0.0", + "@vitest/coverage-v8": "3.2.4", + "@vitest/ui": "3.2.4", + "autoprefixer": "10.4.24", + "dotenv": "17.2.3", + "eslint": "9.39.2", "eslint-config-next": "15.5.10", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-unused-imports": "^4.2.0", - "jsdom": "^27.0.0", - "knip": "^5.83.1", - "postcss": "^8.4.33", + "eslint-config-prettier": "10.1.8", + "eslint-plugin-unused-imports": "4.3.0", + "jsdom": "27.4.0", + "knip": "5.83.1", + "postcss": "8.5.6", "prettier": "3.2.5", - "tailwindcss": "^3.4.1", + "tailwindcss": "3.4.19", "typescript": "5.9.3", - "vite": "^7.1.11", - "vitest": "^3.2.4" + "vite": "7.3.1", + "vitest": "3.2.4" }, "overrides": { "prismjs": "1.30.0",