diff --git a/.github/workflows/check-lazy-openapi-snapshot.yml b/.github/workflows/check-lazy-openapi-snapshot.yml new file mode 100644 index 0000000000..2e4ed3637f --- /dev/null +++ b/.github/workflows/check-lazy-openapi-snapshot.yml @@ -0,0 +1,75 @@ +name: Check Lazy OpenAPI Snapshot + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - "litellm_**" + +permissions: + contents: read + checks: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: uv sync --frozen --all-groups --all-extras + + - name: Regenerate snapshot to /tmp + id: regen + run: | + cp litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.committed.json + uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot + mv litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.fresh.json + mv /tmp/snapshot.committed.json litellm/proxy/_lazy_openapi_snapshot.json + + - name: Compare + id: diff + continue-on-error: true + run: | + diff -q /tmp/snapshot.fresh.json litellm/proxy/_lazy_openapi_snapshot.json + + - name: Mark neutral if drift + if: steps.diff.outcome == 'failure' + uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + name: lazy-openapi-snapshot + conclusion: neutral + output: | + { + "title": "Lazy openapi snapshot is stale", + "summary": "Run `python -m litellm.proxy._lazy_openapi_snapshot` and commit the regenerated `litellm/proxy/_lazy_openapi_snapshot.json`. Not blocking — the snapshot will regenerate at release if not committed." + } diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml index 13b76c94df..ec2651306f 100644 --- a/.github/workflows/create-release-branch.yml +++ b/.github/workflows/create-release-branch.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: tag: - description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/" + description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/" required: true type: string commit_hash: @@ -14,7 +14,7 @@ on: workflow_call: inputs: tag: - description: "Release tag" + description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)" required: true type: string commit_hash: @@ -40,8 +40,8 @@ jobs: echo "::error::commit_hash must be a full 40-character commit SHA" exit 1 fi - if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then - echo "::error::tag must start with vX.Y.Z" + if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable" exit 1 fi diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 68ab397d82..39d078267f 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: tag: - description: "Release tag (e.g. v1.83.0-stable)" + description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)" required: true type: string commit_hash: @@ -30,8 +30,8 @@ jobs: echo "::error::commit_hash must be a full 40-character commit SHA" exit 1 fi - if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then - echo "::error::tag must start with vX.Y.Z" + if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable" exit 1 fi @@ -45,6 +45,11 @@ jobs: const tag = process.env.TAG; const commitHash = process.env.COMMIT_HASH; + // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases. + // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]` + // are stable maintenance releases, not pre-releases. + const isPrerelease = /(?:rc|nightly|alpha|beta|\.dev)/i.test(tag); + const cosignSection = [ `## Verify Docker Image Signature`, ``, @@ -89,7 +94,7 @@ jobs: target_commitish: commitHash, name: tag, owner: context.repo.owner, - prerelease: false, + prerelease: isPrerelease, repo: context.repo.repo, tag_name: tag, }); diff --git a/.npmrc b/.npmrc index 168e81a1c4..7999681cc3 100644 --- a/.npmrc +++ b/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc index 168e81a1c4..7999681cc3 100644 --- a/litellm-js/proxy/.npmrc +++ b/litellm-js/proxy/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc index 168e81a1c4..7999681cc3 100644 --- a/litellm-js/spend-logs/.npmrc +++ b/litellm-js/spend-logs/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql new file mode 100644 index 0000000000..bffdaaebc5 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql @@ -0,0 +1,2 @@ +-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores). +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql new file mode 100644 index 0000000000..6454f26765 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql @@ -0,0 +1,75 @@ +-- CreateTable +CREATE TABLE "LiteLLM_WorkflowRun" ( + "run_id" TEXT NOT NULL, + "session_id" TEXT NOT NULL, + "workflow_type" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'pending', + "created_by" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + "input" JSONB, + "output" JSONB, + "metadata" JSONB, + + CONSTRAINT "LiteLLM_WorkflowRun_pkey" PRIMARY KEY ("run_id") +); + +-- CreateTable +CREATE TABLE "LiteLLM_WorkflowEvent" ( + "event_id" TEXT NOT NULL, + "run_id" TEXT NOT NULL, + "event_type" TEXT NOT NULL, + "step_name" TEXT NOT NULL, + "sequence_number" INTEGER NOT NULL, + "data" JSONB, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_WorkflowEvent_pkey" PRIMARY KEY ("event_id") +); + +-- CreateTable +CREATE TABLE "LiteLLM_WorkflowMessage" ( + "message_id" TEXT NOT NULL, + "run_id" TEXT NOT NULL, + "role" TEXT NOT NULL, + "content" TEXT NOT NULL, + "sequence_number" INTEGER NOT NULL, + "session_id" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_WorkflowMessage_pkey" PRIMARY KEY ("message_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_WorkflowRun_session_id_key" ON "LiteLLM_WorkflowRun"("session_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowRun_workflow_type_status_idx" ON "LiteLLM_WorkflowRun"("workflow_type", "status"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowRun_session_id_idx" ON "LiteLLM_WorkflowRun"("session_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowRun_created_at_idx" ON "LiteLLM_WorkflowRun"("created_at"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowRun_created_by_idx" ON "LiteLLM_WorkflowRun"("created_by"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowEvent_run_id_idx" ON "LiteLLM_WorkflowEvent"("run_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_WorkflowEvent_run_id_sequence_number_key" ON "LiteLLM_WorkflowEvent"("run_id", "sequence_number"); + +-- CreateIndex +CREATE INDEX "LiteLLM_WorkflowMessage_run_id_idx" ON "LiteLLM_WorkflowMessage"("run_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_WorkflowMessage_run_id_sequence_number_key" ON "LiteLLM_WorkflowMessage"("run_id", "sequence_number"); + +-- AddForeignKey +ALTER TABLE "LiteLLM_WorkflowEvent" ADD CONSTRAINT "LiteLLM_WorkflowEvent_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "LiteLLM_WorkflowMessage" ADD CONSTRAINT "LiteLLM_WorkflowMessage_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8f07c5afa3..a9d3911c07 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable { models String[] @default([]) blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user + search_tools String[] @default([]) // search_tool_name values this key/team/user may call teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] @@ -1290,3 +1291,80 @@ model LiteLLM_AdaptiveRouterSession { @@id([session_id, router_name, model_name]) @@index([last_activity_at], map: "idx_adaptive_router_session_activity") } + +// --------------------------------------------------------------------------- +// Workflow Run Tracking +// +// Generic durable state tracking for any agent or automated workflow. +// Design: three tables — run (header + materialized status), event (append-only +// source of truth for state transitions), message (conversation inbox/outbox). +// +// Usage: +// - Set `workflow_type` to identify the owning system (e.g. "shin-builder"). +// - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.). +// - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to +// the proxy — all spend logs for this run are automatically tagged. +// --------------------------------------------------------------------------- + +// One instance of work being done. `status` is a materialized cache of the +// latest event; the event log is the authoritative source of truth. +model LiteLLM_WorkflowRun { + run_id String @id @default(uuid()) + session_id String @unique @default(uuid()) + workflow_type String + status String @default("pending") + created_by String? // user_id of the key that created this run; null = created by master key + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + input Json? + output Json? + metadata Json? + + events LiteLLM_WorkflowEvent[] + messages LiteLLM_WorkflowMessage[] + + @@index([workflow_type, status]) + @@index([session_id]) + @@index([created_at]) + @@index([created_by]) +} + +// Append-only log of state transitions. Never mutate rows here. +// `step_name` and `event_type` are caller-defined strings — no hardcoded enums. +// Status auto-update rules (applied by the append endpoint): +// step.started → run.status = running +// step.failed → run.status = failed +// hook.waiting → run.status = paused +// hook.received → run.status = running +model LiteLLM_WorkflowEvent { + event_id String @id @default(uuid()) + run_id String + event_type String + step_name String + sequence_number Int + data Json? + created_at DateTime @default(now()) + + run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id]) + + @@unique([run_id, sequence_number]) + @@index([run_id]) +} + +// Conversation inbox/outbox — full message content, separate from the durable +// event log. Spend logs truncate messages; this table stores them in full. +// `session_id` here is the Claude --resume session ID (or similar). +model LiteLLM_WorkflowMessage { + message_id String @id @default(uuid()) + run_id String + role String + content String + sequence_number Int + session_id String? + created_at DateTime @default(now()) + + run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id]) + + @@unique([run_id, sequence_number]) + @@index([run_id]) +} diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 41c78296fb..b8710da343 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.69" +version = "0.4.70" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.69" +version = "0.4.70" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 23f444b1ce..259439d4d0 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config( ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( - custom_llm_provider - ), + message=( + "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. " + "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. " + "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded." + ).format(custom_llm_provider), model="n/a", llm_provider=custom_llm_provider, response=httpx.Response( status_code=400, content="Unsupported provider", - request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"), # type: ignore ), ) return response diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 34ae3638a5..6115a444ce 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -392,6 +392,7 @@ class DualCache(BaseCache): value: float, parent_otel_span: Optional[Span] = None, local_only: bool = False, + refresh_ttl: bool = False, **kwargs, ) -> Optional[float]: """ @@ -399,6 +400,9 @@ class DualCache(BaseCache): Value - float - the value you want to increment by + Refresh_ttl - bool - if True, resets the Redis TTL on every write. + Default False preserves window-style semantics. + Returns - the incremented value, or None if no cache backend is available (in_memory_cache is None and Redis failed/is absent). """ @@ -415,6 +419,7 @@ class DualCache(BaseCache): value, parent_otel_span=parent_otel_span, ttl=kwargs.get("ttl", None), + refresh_ttl=refresh_ttl, ) return result diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 84a2887f52..deee4f6ea4 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -824,6 +824,7 @@ class RedisCache(BaseCache): value: float, ttl: Optional[int] = None, parent_otel_span: Optional[Span] = None, + refresh_ttl: bool = False, ) -> float: from redis.asyncio import Redis @@ -834,11 +835,12 @@ class RedisCache(BaseCache): try: result = await _redis_client.incrbyfloat(name=key, amount=value) if _used_ttl is not None: - # check if key already has ttl, if not -> set ttl - current_ttl = await _redis_client.ttl(key) - if current_ttl == -1: - # Key has no expiration + if refresh_ttl: await _redis_client.expire(key, _used_ttl) + else: + current_ttl = await _redis_client.ttl(key) + if current_ttl == -1: + await _redis_client.expire(key, _used_ttl) ## LOGGING ## end_time = time.time() diff --git a/litellm/constants.py b/litellm/constants.py index fc9f5730cd..6c889a317b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -224,6 +224,16 @@ AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int( ) AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120)) AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) +# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs +# whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT +# Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing +# during a long provider call and the NAT reaps the flow before the response +# arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset +# the NAT idle timer. +AIOHTTP_SO_KEEPALIVE = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true" +AIOHTTP_TCP_KEEPIDLE = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60)) +AIOHTTP_TCP_KEEPINTVL = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30)) +AIOHTTP_TCP_KEEPCNT = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5)) # enable_cleanup_closed is only needed for Python versions with the SSL leak bug # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960) # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78 @@ -1383,6 +1393,10 @@ except (ValueError, TypeError): LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check" LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli" LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs" +# Stable identifier substituted in place of the master key on UserAPIKeyAuth +# objects so the master key (or its hash) never propagates to spend logs, +# Prometheus metrics, audit trails, or any other downstream consumer. +LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key" # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") @@ -1411,6 +1425,7 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id" LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli" LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token" CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session" +CLI_SSO_SESSION_TTL_SECONDS = 600 CLI_JWT_TOKEN_NAME = "cli-jwt-token" # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility CLI_JWT_EXPIRATION_HOURS = int( diff --git a/litellm/google_genai/streaming_iterator.py b/litellm/google_genai/streaming_iterator.py index 8cb2ee0937..3e97b48077 100644 --- a/litellm/google_genai/streaming_iterator.py +++ b/litellm/google_genai/streaming_iterator.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.success_handler import ( @@ -29,12 +29,14 @@ class BaseGoogleGenAIGenerateContentStreamingIterator: litellm_logging_obj: LiteLLMLoggingObj, request_body: dict, model: str, + hidden_params: Optional[Dict[str, Any]] = None, ): self.litellm_logging_obj = litellm_logging_obj self.request_body = request_body self.start_time = datetime.now() self.collected_chunks: List[bytes] = [] self.model = model + self._hidden_params: Dict[str, Any] = hidden_params or {} async def _handle_async_streaming_logging( self, @@ -76,11 +78,13 @@ class GoogleGenAIGenerateContentStreamingIterator( litellm_metadata: dict, custom_llm_provider: str, request_body: Optional[dict] = None, + hidden_params: Optional[Dict[str, Any]] = None, ): super().__init__( litellm_logging_obj=logging_obj, request_body=request_body or {}, model=model, + hidden_params=hidden_params, ) self.response = response self.model = model @@ -130,11 +134,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator( litellm_metadata: dict, custom_llm_provider: str, request_body: Optional[dict] = None, + hidden_params: Optional[Dict[str, Any]] = None, ): super().__init__( litellm_logging_obj=logging_obj, request_body=request_body or {}, model=model, + hidden_params=hidden_params, ) self.response = response self.model = model diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 71da650dc4..9c626aea84 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -87,9 +87,7 @@ class PromptManagementBase(ABC): try: messages = compiled_prompt_client["prompt_template"] + client_messages except Exception as e: - raise ValueError( - f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}" - ) + raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}") compiled_prompt_client["completed_messages"] = messages return compiled_prompt_client @@ -116,9 +114,7 @@ class PromptManagementBase(ABC): try: messages = compiled_prompt_client["prompt_template"] + client_messages except Exception as e: - raise ValueError( - f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}" - ) + raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}") compiled_prompt_client["completed_messages"] = messages return compiled_prompt_client diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 95bcd4d718..c0ca6835ee 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -1,4 +1,5 @@ from typing import Optional, Tuple +from urllib.parse import urlparse import litellm from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH @@ -8,6 +9,43 @@ from litellm.secret_managers.main import get_secret, get_secret_str from ..types.router import LiteLLM_Params +def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool: + """ + Match a registered openai-compatible endpoint against a caller-supplied + ``api_base`` using parsed-URL semantics, not unanchored substring search. + + Both inputs may be a bare hostname (``api.perplexity.ai``), host+path + (``api.deepinfra.com/v1/openai``), or a full URL + (``https://api.cerebras.ai/v1``). Hostnames must match exactly + (case-insensitive); if the registered endpoint has a non-trivial path, + the api_base path must start with it on a segment boundary. + + The naive ``endpoint in api_base`` shape lets a caller pass + ``https://attacker.com/api.groq.com/openai/v1`` to coerce the proxy + into reading the server's GROQ_API_KEY from the environment and + forwarding it to the attacker's host as a Bearer credential. + """ + + def _parse(value: str): + # Ensure urlparse sees a scheme so it populates hostname / path. + normalized = value if "://" in value else f"https://{value}" + return urlparse(normalized) + + parsed_endpoint = _parse(endpoint) + parsed_url = _parse(api_base) + + endpoint_host = (parsed_endpoint.hostname or "").lower() + url_host = (parsed_url.hostname or "").lower() + if not endpoint_host or endpoint_host != url_host: + return False + + endpoint_path = parsed_endpoint.path.rstrip("/") + if not endpoint_path: + return True + url_path = parsed_url.path.rstrip("/") + return url_path == endpoint_path or url_path.startswith(endpoint_path + "/") + + def _is_non_openai_azure_model(model: str) -> bool: try: model_name = model.split("/", 1)[1] @@ -210,7 +248,7 @@ def get_llm_provider( # noqa: PLR0915 # check if api base is a known openai compatible endpoint if api_base: for endpoint in litellm.openai_compatible_endpoints: - if endpoint in api_base: + if _endpoint_matches_api_base(endpoint, api_base): if endpoint == "api.perplexity.ai": custom_llm_provider = "perplexity" dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY") @@ -348,6 +386,7 @@ def get_llm_provider( # noqa: PLR0915 or "ft:gpt-3.5-turbo" in model or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o or model in litellm.openai_image_generation_models + or model.startswith("gpt-image") or model in litellm.openai_video_generation_models ): custom_llm_provider = "openai" diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 563609af1e..ffb6436f38 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None: ) +def validate_no_callback_env_reference( + param: str, value: object, *, source: str +) -> None: + if _is_env_reference(value): + _raise_env_reference_error(param, source=source) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params = [ "langfuse_public_key", @@ -66,8 +73,9 @@ def initialize_standard_callback_dynamic_params( for param in _supported_callback_params: if param in kwargs: _param_value = kwargs.get(param) - if _is_env_reference(_param_value): - _raise_env_reference_error(param, source="request body") + validate_no_callback_env_reference( + param, _param_value, source="request body" + ) standard_callback_dynamic_params[param] = _param_value # type: ignore # 2. Fallback: check "metadata" or "litellm_params" -> "metadata" @@ -80,8 +88,9 @@ def initialize_standard_callback_dynamic_params( for param in _supported_callback_params: if param not in standard_callback_dynamic_params and param in metadata: _param_value = metadata.get(param) - if _is_env_reference(_param_value): - _raise_env_reference_error(param, source="metadata") + validate_no_callback_env_reference( + param, _param_value, source="metadata" + ) standard_callback_dynamic_params[param] = _param_value # type: ignore return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 888999504f..59d0465e6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -982,9 +982,9 @@ class CostCalculatorUtils: image_response=completion_response, ) elif custom_llm_provider == litellm.LlmProviders.OPENAI.value: - # Check if this is a gpt-image model (token-based pricing) + # gpt-image models use token-based pricing. model_lower = model.lower() - if "gpt-image-1" in model_lower: + if "gpt-image" in model_lower: from litellm.llms.openai.image_generation.cost_calculator import ( cost_calculator as openai_gpt_image_cost_calculator, ) @@ -1004,9 +1004,9 @@ class CostCalculatorUtils: optional_params=optional_params, ) elif custom_llm_provider == litellm.LlmProviders.AZURE.value: - # Check if this is a gpt-image model (token-based pricing) + # gpt-image models use token-based pricing. model_lower = model.lower() - if "gpt-image-1" in model_lower: + if "gpt-image" in model_lower: from litellm.llms.openai.image_generation.cost_calculator import ( cost_calculator as openai_gpt_image_cost_calculator, ) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 78378faa26..5fd42fe0d3 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -824,8 +824,6 @@ def convert_to_model_response_object( # noqa: PLR0915 stream=stream, start_time=start_time, end_time=end_time, - hidden_params=hidden_params, - _response_headers=_response_headers, convert_tool_call_to_json_mode=convert_tool_call_to_json_mode, ) raise Exception( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fe8387476e..3a83162fb2 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: return sanitized +_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"} + + +def _is_anthropic_document_data_uri(url: str) -> bool: + # Anthropic's base64 document source accepts only application/pdf and + # text/plain (see select_anthropic_content_block_type_for_file). Routing + # other mimes here would produce a document block the API rejects, so we + # leave them on the image code path. + match = re.match(r"data:([^;,]+)", url) + if not match: + return False + return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + + def convert_to_anthropic_tool_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], force_base64: bool = False, @@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result( """ anthropic_content: Union[ str, - List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]], + List[ + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] + ], ] = "" if isinstance(message["content"], str): anthropic_content = message["content"] elif isinstance(message["content"], List): content_list = message["content"] anthropic_content_list: List[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ] = [] for content in content_list: if content["type"] == "text": @@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result( text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": + image_url_value = content["image_url"] format = ( - content["image_url"].get("format") - if isinstance(content["image_url"], dict) + image_url_value.get("format") + if isinstance(image_url_value, dict) else None ) - _anthropic_image_param = create_anthropic_image_param( - content["image_url"], format=format, is_bedrock_invoke=force_base64 + url_str = ( + image_url_value.get("url") + if isinstance(image_url_value, dict) + else image_url_value ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, + # Data URIs with non-image mime types (e.g. application/pdf) must + # translate to Anthropic document blocks, not image blocks — + # wrapping a PDF in `type: "image"` is rejected by the API. + if isinstance(url_str, str) and _is_anthropic_document_data_uri( + url_str + ): + synth_file_message: ChatCompletionFileObject = { + "type": "file", + "file": {"file_data": url_str}, + } + _document_block = anthropic_process_openai_file_message( + synth_file_message + ) + _document_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _document_block + ), + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesDocumentParam, _document_block) + ) + else: + _anthropic_image_param = create_anthropic_image_param( + image_url_value, + format=format, + is_bedrock_invoke=force_base64, + ) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesImageParam, _anthropic_image_param) + ) + elif content["type"] == "file": + file_content = cast(ChatCompletionFileObject, content) + _file_block = anthropic_process_openai_file_message(file_content) + _file_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _file_block + ), original_content_element=content, ) - anthropic_content_list.append( - cast(AnthropicMessagesImageParam, _anthropic_image_param) - ) + anthropic_content_list.append(_file_block) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None @@ -3977,6 +4042,55 @@ def _convert_to_bedrock_tool_call_result( tool_result_content_blocks.append( BedrockToolResultContentBlock(image=_block["image"]) ) + elif "document" in _block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_block["document"]) + ) + else: + verbose_logger.warning( + "Bedrock Converse: unrecognized BedrockContentBlock keys " + "%s for image_url tool-result block %s; dropping.", + list(_block.keys()), + content, + ) + elif content["type"] == "file": + # Match the user-message path (_process_file_message): accept + # either file_data (base64 data URI) or file_id (server-side + # reference / URL) and hand off to BedrockImageProcessor. Raise + # BadRequestError on both-None rather than silently dropping. + file_obj = content.get("file") or {} + file_data = file_obj.get("file_data") + file_id = file_obj.get("file_id") + if file_data is None and file_id is None: + raise litellm.BadRequestError( + message="file_data and file_id cannot both be None. Got={}".format( + content + ), + model="", + llm_provider="bedrock", + ) + file_format = file_obj.get("format") + _file_block: BedrockContentBlock = ( + BedrockImageProcessor.process_image_sync( + image_url=cast(str, file_id or file_data), + format=file_format, + ) + ) + if "document" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_file_block["document"]) + ) + elif "image" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=_file_block["image"]) + ) + else: + verbose_logger.warning( + "Bedrock Converse: unrecognized BedrockContentBlock keys " + "%s for file tool-result block %s; dropping.", + list(_file_block.keys()), + content, + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index f3f560b33b..dbc9cabdc7 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -60,6 +60,9 @@ def _redact_choice_content(choice): def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: + if hasattr(output_item, "text"): + output_item.text = "redacted-by-litellm" + if hasattr(output_item, "content") and isinstance(output_item.content, list): for content_part in output_item.content: if hasattr(content_part, "text"): @@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_responses_api_output_dict(output_items, redacted_str: str): + """Helper to redact ResponsesAPIResponse output items in dict form.""" + for output_item in output_items: + if not isinstance(output_item, dict): + continue + + if "text" in output_item: + output_item["text"] = redacted_str + + if isinstance(output_item.get("content"), list): + for content_item in output_item["content"]: + if isinstance(content_item, dict) and "text" in content_item: + content_item["text"] = redacted_str + + if output_item.get("type") == "reasoning" and isinstance( + output_item.get("summary"), list + ): + for summary_item in output_item["summary"]: + if isinstance(summary_item, dict) and "text" in summary_item: + summary_item["text"] = redacted_str + + def _redact_standard_logging_object(model_call_details: dict): """Redact messages and response inside standard_logging_object if present.""" standard_logging_object = model_call_details.get("standard_logging_object") @@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict): if isinstance(response, dict) and "output" in response: # ResponsesAPIResponse format - redact content in output items if isinstance(response.get("output"), list): - for output_item in response["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str + _redact_responses_api_output_dict(response["output"], redacted_str) elif isinstance(response, dict) and "choices" in response: # ModelResponse dict format - redact content in choices if isinstance(response.get("choices"), list): - for choice in response["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = redacted_str - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = redacted_str - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None + _redact_model_response_dict_choices(response["choices"], redacted_str) elif isinstance(response, str): standard_logging_object["response"] = redacted_str else: @@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict): standard_logging_object["response"] = {"text": redacted_str} +def _redact_model_response_dict_choices(choices, redacted_str: str): + for choice in choices: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "reasoning_content" in choice["message"]: + choice["message"]["reasoning_content"] = redacted_str + if "thinking_blocks" in choice["message"]: + choice["message"]["thinking_blocks"] = None + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "reasoning_content" in choice["delta"]: + choice["delta"]["reasoning_content"] = redacted_str + if "thinking_blocks" in choice["delta"]: + choice["delta"]["thinking_blocks"] = None + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + else: + _redact_choice_content(choice) + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result): ] model_call_details["prompt"] = "" model_call_details["input"] = "" + _redact_standard_logging_object(model_call_details) # Redact streaming response if ( @@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, dict) and "choices" in _result: # Handle dict representation of ModelResponse (e.g., from model_dump()) if _result.get("choices") is not None: - for choice in _result["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["message"]: - choice["message"][ - "reasoning_content" - ] = "redacted-by-litellm" - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["delta"]: - choice["delta"][ - "reasoning_content" - ] = "redacted-by-litellm" - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - else: - _redact_choice_content(choice) + _redact_model_response_dict_choices( + _result["choices"], "redacted-by-litellm" + ) + elif isinstance(_result, dict) and "output" in _result: + if isinstance(_result.get("output"), list): + _redact_responses_api_output_dict( + _result["output"], "redacted-by-litellm" + ) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 8b88ef9482..d7803455b4 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -21,6 +21,8 @@ class SensitiveDataMasker: "auth", "authorization", "credential", + # Plural form: Vertex uses ``vertex_credentials``; segment-exact + # matching otherwise misses it because "credential" != "credentials". "credentials", "access", "private", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 31b9bad339..61ddf801a2 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1553,25 +1553,43 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) data["output_config"] = output_config - def _transform_response_for_json_mode( + def _resolve_json_mode_non_streaming( self, json_mode: Optional[bool], tool_calls: List[ChatCompletionToolCallChunk], - ) -> Optional[LitellmMessage]: - _message: Optional[LitellmMessage] = None - if json_mode is True and len(tool_calls) == 1: - # check if tool name is the default tool name - json_mode_content_str: Optional[str] = None - if ( - "name" in tool_calls[0]["function"] - and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME - ): - json_mode_content_str = tool_calls[0]["function"].get("arguments") - if json_mode_content_str is not None: - _message = AnthropicConfig._convert_tool_response_to_message( - tool_calls=tool_calls, - ) - return _message + ) -> Tuple[ + Optional[LitellmMessage], + List[ChatCompletionToolCallChunk], + Optional[str], + ]: + """Strip internal response_format tool calls; merge payload into content when mixed with user tools.""" + if json_mode is not True or not tool_calls: + return None, tool_calls, None + + json_indices = [ + i + for i, t in enumerate(tool_calls) + if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + if not json_indices: + return None, tool_calls, None + + if len(json_indices) == len(tool_calls): + json_tool = tool_calls[json_indices[0]] + if json_tool.get("function", {}).get("arguments") is None: + return None, tool_calls, None + _message = AnthropicConfig._convert_tool_response_to_message( + tool_calls=[json_tool] + ) + return _message, [], None + + first_json = tool_calls[json_indices[0]] + json_msg = AnthropicConfig._convert_tool_response_to_message([first_json]) + extra_content: Optional[str] = ( + json_msg.content if json_msg is not None else None + ) + filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices] + return None, filtered_tools, extra_content def extract_response_content(self, completion_response: dict) -> Tuple[ str, @@ -1931,19 +1949,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_calls, ) + json_mode_message, tool_calls_for_message, json_extra_content = ( + self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + ) + merged_text = text_content or "" + if json_extra_content: + merged_text = ( + merged_text + json_extra_content if merged_text else json_extra_content + ) + _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, + tool_calls=tool_calls_for_message, + content=merged_text or None, provider_specific_fields=provider_specific_fields, thinking_blocks=thinking_blocks, reasoning_content=reasoning_content, ) _message.provider_specific_fields = provider_specific_fields - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) if json_mode_message is not None: completion_response["stop_reason"] = "stop" _message = json_mode_message diff --git a/litellm/llms/azure/image_generation/__init__.py b/litellm/llms/azure/image_generation/__init__.py index fcdf49f291..a9cf151464 100644 --- a/litellm/llms/azure/image_generation/__init__.py +++ b/litellm/llms/azure/image_generation/__init__.py @@ -24,6 +24,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig: return AzureDallE3ImageGenerationConfig() else: verbose_logger.debug( - f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format." + f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format." ) return AzureGPTImageGenerationConfig() diff --git a/litellm/llms/azure/image_generation/gpt_transformation.py b/litellm/llms/azure/image_generation/gpt_transformation.py index 1f5f65f693..2d46592e3f 100644 --- a/litellm/llms/azure/image_generation/gpt_transformation.py +++ b/litellm/llms/azure/image_generation/gpt_transformation.py @@ -3,7 +3,7 @@ from litellm.llms.openai.image_generation import GPTImageGenerationConfig class AzureGPTImageGenerationConfig(GPTImageGenerationConfig): """ - Azure gpt-image-1 image generation config + Azure gpt-image image generation config """ pass diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index b62acb6516..d1b93c9e7a 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -95,6 +95,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 7874201f7f..166f876ba0 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -33,6 +33,7 @@ class BaseRerankConfig(ABC): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: return {} diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 5fbf0a4b19..85a9c83826 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -59,6 +59,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: pass @@ -70,6 +71,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """ Optional async version of transform_search_vector_store_request. @@ -84,6 +86,7 @@ class BaseVectorStoreConfig: api_base=api_base, litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, + extra_body=extra_body, ) @abstractmethod diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 4e3521b119..dae60948a5 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1,6 +1,7 @@ import hashlib import json import os +import re import urllib.parse from datetime import datetime from typing import ( @@ -37,6 +38,11 @@ else: AWSPreparedRequest = Any +# Real AWS region names are lowercase letters, digits, and hyphens +# (e.g. "us-east-1", "eu-west-2", "us-gov-west-1", "cn-north-1"). +_VALID_AWS_REGION_PATTERN = re.compile(r"\A[a-z0-9-]+\Z") + + class Boto3CredentialsInfo(BaseModel): credentials: Credentials aws_region_name: str @@ -284,6 +290,9 @@ class BaseAWSLLM: if not region: # Check if region is empty return None + if not _VALID_AWS_REGION_PATTERN.match(region): + return None + return region except Exception: # Catch any unexpected errors and return None @@ -481,6 +490,7 @@ class BaseAWSLLM: str: The AWS region name """ aws_region_name = optional_params.get("aws_region_name", None) + self._validate_aws_region_name(aws_region_name) ### SET REGION NAME ### if aws_region_name is None: # check model arn # @@ -519,8 +529,25 @@ class BaseAWSLLM: except Exception: aws_region_name = "us-west-2" + self._validate_aws_region_name(aws_region_name) return aws_region_name + @staticmethod + def _validate_aws_region_name(aws_region_name: Optional[str]) -> None: + """ + Validate that an AWS region name conforms to the expected format + (lowercase alphanumerics and hyphens). Raises ValueError otherwise. + """ + if aws_region_name is None: + return + if not isinstance(aws_region_name, str) or not _VALID_AWS_REGION_PATTERN.match( + aws_region_name + ): + raise ValueError( + f"Invalid AWS region format: {aws_region_name!r}. " + "Region names must contain only lowercase letters, digits, and hyphens." + ) + def get_aws_region_name_for_non_llm_api_calls( self, aws_region_name: Optional[str] = None, @@ -532,6 +559,7 @@ class BaseAWSLLM: For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars. """ + self._validate_aws_region_name(aws_region_name) if aws_region_name is None: # check env # litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) @@ -549,6 +577,8 @@ class BaseAWSLLM: if aws_region_name is None: aws_region_name = "us-west-2" + + self._validate_aws_region_name(aws_region_name) return aws_region_name @staticmethod diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index a27153365d..61a7d4c08d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1942,8 +1942,8 @@ class AmazonConverseConfig(BaseConfig): completion_response = ConverseResponseBlock(**response.json()) # type: ignore except Exception as e: raise BedrockError( - message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( - response.text, str(e) + message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( + str(e) ), status_code=422, ) diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index 4da0a7c779..f028503c6a 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -1,14 +1,16 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from copy import deepcopy +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urlparse import httpx +from litellm._logging import verbose_logger from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.integrations.rag.bedrock_knowledgebase import ( BedrockKBContent, - BedrockKBResponse, BedrockKBRetrievalConfiguration, + BedrockKBResponse, BedrockKBRetrievalQuery, ) from litellm.types.router import GenericLiteLLMParams @@ -202,6 +204,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: if isinstance(query, list): query = " ".join(query) @@ -213,24 +216,46 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): } retrieval_config: Dict[str, Any] = {} + + if isinstance(extra_body, dict): + retrieval_config = deepcopy( + extra_body.get("retrievalConfiguration") + or extra_body.get("retrieval_configuration") + or {} + ) max_results = vector_store_search_optional_params.get("max_num_results") if max_results is not None: + existing_number_of_results = retrieval_config.get( + "vectorSearchConfiguration", {} + ).get("numberOfResults") + if ( + existing_number_of_results is not None + and existing_number_of_results != max_results + ): + verbose_logger.debug( + "Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.numberOfResults (%s) with max_num_results=%s", + existing_number_of_results, + max_results, + ) retrieval_config.setdefault("vectorSearchConfiguration", {})[ "numberOfResults" ] = max_results filters = vector_store_search_optional_params.get("filters") if filters is not None: + existing_filter = retrieval_config.get("vectorSearchConfiguration", {}).get( + "filter" + ) + if existing_filter is not None and existing_filter != filters: + verbose_logger.debug( + "Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.filter with filters from vector_store_search_optional_params" + ) retrieval_config.setdefault("vectorSearchConfiguration", {})[ "filter" ] = filters if retrieval_config: - # Create a properly typed retrieval configuration - typed_retrieval_config: BedrockKBRetrievalConfiguration = {} - if "vectorSearchConfiguration" in retrieval_config: - typed_retrieval_config["vectorSearchConfiguration"] = retrieval_config[ - "vectorSearchConfiguration" - ] - request_body["retrievalConfiguration"] = typed_retrieval_config + request_body["retrievalConfiguration"] = cast( + BedrockKBRetrievalConfiguration, retrieval_config + ) litellm_logging_obj.model_call_details["query"] = query return url, request_body diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index 531b94d180..64ae8e8ffa 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -111,6 +111,7 @@ class CohereRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Cohere rerank") diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index 60d22ff4be..4c800d6455 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -71,6 +71,7 @@ class CohereRerankV2Config(CohereRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Cohere rerank") diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 03d2af7232..dd955c23d2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1,5 +1,7 @@ import asyncio +import inspect import os +import socket import ssl import sys import time @@ -29,6 +31,10 @@ from litellm.constants import ( AIOHTTP_CONNECTOR_LIMIT_PER_HOST, AIOHTTP_KEEPALIVE_TIMEOUT, AIOHTTP_NEEDS_CLEANUP_CLOSED, + AIOHTTP_SO_KEEPALIVE, + AIOHTTP_TCP_KEEPCNT, + AIOHTTP_TCP_KEEPIDLE, + AIOHTTP_TCP_KEEPINTVL, AIOHTTP_TTL_DNS_CACHE, COMPLETION_HTTP_FALLBACK_SECONDS, DEFAULT_SSL_CIPHERS, @@ -54,6 +60,57 @@ except Exception: version = "0.0.0" +# aiohttp 3.10+ exposes a `socket_factory` kwarg on TCPConnector. Older +# versions don't — detect once and skip the keep-alive wiring there. +# https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.TCPConnector +_AIOHTTP_SUPPORTS_SOCKET_FACTORY = ( + "socket_factory" in inspect.signature(TCPConnector.__init__).parameters +) + + +def _build_aiohttp_keepalive_socket_factory() -> ( + Optional[Callable[[Tuple[Any, ...]], socket.socket]] +): + """ + Build a socket_factory that enables SO_KEEPALIVE on aiohttp TCP sockets. + + Why: by default, aiohttp creates sockets without SO_KEEPALIVE, so the kernel + sends nothing during a long idle TCP connection. NAT/LB hops (e.g. AWS NAT + Gateway, 350s idle timeout) reap the flow well before slow provider + responses (OpenAI/Azure: up to 600s) arrive. Enabling SO_KEEPALIVE makes + the kernel emit TCP probes that reset the NAT idle timer. + + Returns None when AIOHTTP_SO_KEEPALIVE is disabled or aiohttp is too old. + """ + if not AIOHTTP_SO_KEEPALIVE or not _AIOHTTP_SUPPORTS_SOCKET_FACTORY: + return None + + def factory(addr_info: Tuple[Any, ...]) -> socket.socket: + family, type_, proto = addr_info[0], addr_info[1], addr_info[2] + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + # Linux: TCP_KEEPIDLE is idle-before-first-probe. + # macOS/Darwin: TCP_KEEPALIVE is the equivalent. + if hasattr(socket, "TCP_KEEPIDLE"): + sock.setsockopt( + socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, AIOHTTP_TCP_KEEPIDLE + ) + elif hasattr(socket, "TCP_KEEPALIVE"): + sock.setsockopt( + socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, AIOHTTP_TCP_KEEPIDLE + ) + if hasattr(socket, "TCP_KEEPINTVL"): + sock.setsockopt( + socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, AIOHTTP_TCP_KEEPINTVL + ) + if hasattr(socket, "TCP_KEEPCNT"): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, AIOHTTP_TCP_KEEPCNT) + return sock + + return factory + + def get_default_headers() -> dict: """ Get default headers for HTTP requests. @@ -935,6 +992,11 @@ class AsyncHTTPHandler: transport_connector_kwargs["limit_per_host"] = ( AIOHTTP_CONNECTOR_LIMIT_PER_HOST ) + # Returns None when SO_KEEPALIVE is disabled or aiohttp is too old to + # accept socket_factory — version detection lives inside the builder. + socket_factory = _build_aiohttp_keepalive_socket_factory() + if socket_factory is not None: + transport_connector_kwargs["socket_factory"] = socket_factory return LiteLLMAiohttpTransport( client=lambda: ClientSession( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b9ada079f6..dc625918b9 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -155,6 +155,30 @@ else: LiteLLMLoggingObj = Any +def _google_genai_streaming_hidden_params( + *, + api_base: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + response_headers: httpx.Headers, +) -> Dict[str, Any]: + """Pre-stream metadata for proxy response headers (mirrors CustomStreamWrapper._hidden_params).""" + from litellm.litellm_core_utils.core_helpers import process_response_headers + + _model_info: Dict[str, Any] = dict( + getattr(litellm_params, "model_info", None) or {} + ) + _raw_id = _model_info.get("id") or logging_obj.get_router_model_id() or "" + _model_id = _raw_id if isinstance(_raw_id, str) else str(_raw_id) + return { + "model_id": _model_id, + "api_base": api_base, + "cache_key": "", + "response_cost": "", + "additional_headers": process_response_headers(response_headers), + } + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -983,6 +1007,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, api_base: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + litellm_params: Optional[Dict[str, Any]] = None, ) -> RerankResponse: # get config from model, custom llm provider headers = provider_config.validate_environment( @@ -1002,6 +1027,7 @@ class BaseLLMHTTPHandler: model=model, optional_rerank_params=optional_rerank_params, headers=headers, + litellm_params=litellm_params, ) ## LOGGING @@ -2511,10 +2537,16 @@ class BaseLLMHTTPHandler: }, ) + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + try: - response = await async_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + response = await async_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( @@ -2595,10 +2627,16 @@ class BaseLLMHTTPHandler: }, ) + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + try: - response = sync_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + response = sync_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( @@ -8585,6 +8623,7 @@ class BaseLLMHTTPHandler: api_base=api_base, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), + extra_body=extra_body, ) else: ( @@ -8597,6 +8636,7 @@ class BaseLLMHTTPHandler: api_base=api_base, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), + extra_body=extra_body, ) all_optional_params: Dict[str, Any] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -8697,6 +8737,7 @@ class BaseLLMHTTPHandler: api_base=api_base, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), + extra_body=extra_body, ) all_optional_params: Dict[str, Any] = dict(litellm_params) @@ -10425,6 +10466,12 @@ class BaseLLMHTTPHandler: litellm_metadata=litellm_metadata or {}, custom_llm_provider=custom_llm_provider, request_body=data, + hidden_params=_google_genai_streaming_hidden_params( + api_base=api_base, + litellm_params=litellm_params, + logging_obj=logging_obj, + response_headers=response.headers, + ), ) else: response = sync_httpx_client.post( @@ -10534,6 +10581,12 @@ class BaseLLMHTTPHandler: litellm_metadata=litellm_metadata or {}, custom_llm_provider=custom_llm_provider, request_body=data, + hidden_params=_google_genai_streaming_hidden_params( + api_base=api_base, + litellm_params=litellm_params, + logging_obj=logging_obj, + response_headers=response.headers, + ), ) else: response = await async_httpx_client.post( diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 71e300d258..276735f475 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -132,6 +132,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: # Convert OptionalRerankParams to dict as expected by parent class if optional_rerank_params is None: diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index eb92399a05..4a7b64b9b7 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -127,6 +127,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to Fireworks AI rerank format diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index e6e8369643..35d83bd2ad 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -118,6 +118,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 8066e53afc..60b6dc7d23 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -121,6 +121,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Hosted VLLM rerank") diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 3f83b8e422..2c847b617e 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -146,6 +146,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Union[OptionalRerankParams, dict], headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for HuggingFace rerank") diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index 48d876f8ea..ad4416925a 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -74,7 +74,11 @@ class JinaAIRerankConfig(BaseRerankConfig): return cleaned_base def transform_rerank_request( - self, model: str, optional_rerank_params: Dict, headers: Dict + self, + model: str, + optional_rerank_params: Dict, + headers: Dict, + litellm_params: Optional[dict] = None, ) -> Dict: return {"model": model, **optional_rerank_params} diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index fcf5d14db7..867f6d4b1f 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -25,7 +25,6 @@ else: LiteLLMLoggingObj = Any MILVUS_OPTIONAL_PARAMS = { - "dbName", "annsField", "limit", "filter", @@ -33,7 +32,6 @@ MILVUS_OPTIONAL_PARAMS = { "groupingField", "outputFields", "searchParams", - "partitionNames", "consistencyLevel", } @@ -130,6 +128,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API @@ -172,13 +171,21 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): url = f"{api_base}/v2/vectordb/entities/search" # Build the request body for Azure AI Search with vector search - request_body = { + request_body: Dict[str, Any] = { "collectionName": index_name, "data": [query_vector], "annsField": "book_intro_vector", **vector_store_search_optional_params, } + db_name = litellm_params.get("milvus_db_name") + if db_name: + request_body["dbName"] = db_name + + partition_names = litellm_params.get("milvus_partition_names") + if partition_names: + request_body["partitionNames"] = partition_names + ######################################################### # Update logging object with details of the request ######################################################### diff --git a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py index 757d874bf3..b9a46b8ac2 100644 --- a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py +++ b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py @@ -66,6 +66,7 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request, using clean model name without 'ranking/' prefix. @@ -75,4 +76,5 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig): model=clean_model, optional_rerank_params=optional_rerank_params, headers=headers, + litellm_params=litellm_params, ) diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index bd5abac60c..fc317293ac 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -177,6 +177,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to Nvidia NIM format. diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index 8bca75172f..d009a085fa 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -1,5 +1,5 @@ """ -Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini) +Cost calculator for OpenAI image generation models (gpt-image family) These models use token-based pricing instead of pixel-based pricing like DALL-E. """ @@ -17,13 +17,13 @@ def cost_calculator( custom_llm_provider: Optional[str] = None, ) -> float: """ - Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models. + Calculate cost for OpenAI gpt-image models. Uses the same usage format as Responses API, so we reuse the helper to transform to chat completion format and use generic_cost_per_token. Args: - model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini") + model: The model name (e.g., "gpt-image-1", "gpt-image-2") image_response: The ImageResponse containing usage data custom_llm_provider: Optional provider name diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index c106d7f17b..68f799e574 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: class GPTImageGenerationConfig(BaseImageGenerationConfig): """ - OpenAI gpt-image-1 image generation config + OpenAI gpt-image image generation config """ def get_supported_openai_params( diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index c763ed1c8d..2c11d13748 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -106,6 +106,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: url = f"{api_base}/{vector_store_id}/search" typed_request_body = VectorStoreSearchRequest( diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 275c352b39..5dd1247001 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -101,5 +101,10 @@ "param_mappings": { "max_completion_tokens": "max_tokens" } + }, + "aihubmix": { + "base_url": "https://aihubmix.com/v1", + "api_key_env": "AIHUBMIX_API_KEY", + "api_base_env": "AIHUBMIX_API_BASE" } } diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index ba87a8f2b0..7b22edd867 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -80,6 +80,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: url = f"{api_base}/{vector_store_id}/search" _, request_body = super().transform_search_vector_store_request( @@ -89,5 +90,6 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): api_base=api_base, litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, + extra_body=extra_body, ) return url, request_body diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index ed5397eef0..3238d3e9c1 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -102,6 +102,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError( diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 19b5976986..8270e99d45 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -79,6 +79,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """Sync version - generates embedding synchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -140,6 +141,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """Async version - generates embedding asynchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 028e02eb0c..7436bfef58 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -4,6 +4,7 @@ from typing import Any, Coroutine, Dict, Optional, Union import httpx import litellm +from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -224,8 +225,14 @@ class VertexAIBatchPrediction(VertexLLM): }, ) - response = sync_handler.get( - url=api_base, + # ``api_base`` here can come from caller-supplied request kwargs + # (clientside override). Wrap the fetch in ``safe_get`` so DNS + # rebind / private / cloud-metadata targets are rejected; the + # proxy auth gate already blocks malicious clientside ``api_base`` + # at the boundary — this is defense-in-depth for SDK callers. + response = safe_get( + sync_handler, + api_base, headers=headers, ) @@ -270,8 +277,13 @@ class VertexAIBatchPrediction(VertexLLM): }, ) - response = await client.get( - url=api_base, + # Mirror the sync path: ``api_base`` may come from caller-supplied + # request kwargs, so wrap the fetch in ``async_safe_get`` to reject + # DNS-rebind / private / cloud-metadata targets. Defense-in-depth + # behind the proxy auth gate's clientside ``api_base`` check. + response = await async_safe_get( + client, + api_base, headers=headers, ) if response.status_code != 200: diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index b4bfde5f54..fae175612b 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -27,6 +27,53 @@ class VertexAIError(BaseLLMException): super().__init__(message=message, status_code=status_code, headers=headers) +def vertex_request_labels_from_litellm_params( + litellm_params: Optional[dict], +) -> Optional[Dict[str, str]]: + """ + Build Vertex/GCP billing labels from LiteLLM user metadata on ``litellm_params``: + ``metadata`` (``completion(..., metadata=...)``) or ``litellm_metadata``, + using ``requester_metadata`` string key-value pairs (same convention as Gemini). + ``metadata`` is tried first when both are present. + """ + if not litellm_params: + return None + for key in ("metadata", "litellm_metadata"): + if key not in litellm_params: + continue + metadata = litellm_params[key] + if metadata is None or not isinstance(metadata, dict): + continue + if "requester_metadata" not in metadata: + continue + rm = metadata["requester_metadata"] + if not isinstance(rm, dict): + continue + labels = {k: v for k, v in rm.items() if isinstance(v, str)} + if labels: + return labels + return None + + +def pop_vertex_request_labels( + optional_params: Optional[dict], + litellm_params: Optional[dict], +) -> Optional[Dict[str, str]]: + """ + Resolve labels from optional ``labels`` (Gemini-style) and/or + ``litellm_params["metadata"]`` / ``litellm_params["litellm_metadata"]`` + (``requester_metadata``). Pops ``labels`` from optional_params when present. + """ + labels: Optional[Dict[str, str]] = None + if optional_params is not None and "labels" in optional_params: + raw = optional_params.pop("labels") + if isinstance(raw, dict): + labels = {k: v for k, v in raw.items() if isinstance(v, str)} + if not labels: + labels = vertex_request_labels_from_litellm_params(litellm_params) + return labels if labels else None + + class VertexAIModelRoute(str, Enum): """Enum for Vertex AI model routing""" diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 533bd06d2d..87bd484382 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( response_schema_prompt, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.files import ( get_file_mime_type_for_file_type, get_file_type_from_extension, @@ -714,16 +715,8 @@ def _transform_request_body( # noqa: PLR0915 optional_params.pop("output_config", None) config_fields = GenerationConfig.__annotations__.keys() - # If the LiteLLM client sends Gemini-supported parameter "labels", add it - # as "labels" field to the request sent to the Gemini backend. - labels: Optional[dict[str, str]] = optional_params.pop("labels", None) - # If the LiteLLM client sends OpenAI-supported parameter "metadata", add it - # as "labels" field to the request sent to the Gemini backend. - if labels is None and "metadata" in litellm_params: - metadata = litellm_params["metadata"] - if metadata is not None and "requester_metadata" in metadata: - rm = metadata["requester_metadata"] - labels = {k: v for k, v in rm.items() if isinstance(v, str)} + # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata) + labels = pop_vertex_request_labels(optional_params, litellm_params) filtered_params = { k: v diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index a90e05919f..474ddb402a 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2395,8 +2395,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): completion_response = GenerateContentResponseBody(**raw_response.json()) # type: ignore except Exception as e: raise VertexAIError( - message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( - raw_response.text, str(e) + message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( + str(e) ), status_code=422, headers=raw_response.headers, @@ -2530,8 +2530,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): except Exception as e: raise VertexAIError( - message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( - completion_response, str(e) + message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( + str(e) ), status_code=422, headers=raw_response.headers, diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 1c7696d55a..05ebd685d9 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -7,7 +7,10 @@ import litellm from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) -from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_base_url, + pop_vertex_request_labels, +) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -203,13 +206,16 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): "sampleCount": 1, } - # Merge with optional params + labels = pop_vertex_request_labels(optional_params, litellm_params) + # Merge with optional params (after popping labels so they are not sent as Imagen parameters) parameters = {**default_params, **optional_params} - request_body = { + request_body: dict = { "instances": [{"prompt": prompt}], "parameters": parameters, } + if labels: + request_body["labels"] = labels return request_body diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index 5365183967..3b84972e94 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -11,12 +11,15 @@ import httpx import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.vertex_ai.common_utils import ( + vertex_request_labels_from_litellm_params, +) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( + RerankBilledUnits, RerankResponse, RerankResponseMeta, - RerankBilledUnits, RerankResponseResult, ) @@ -109,6 +112,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform the request from Cohere format to Vertex AI Discovery Engine format @@ -145,6 +149,10 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): # When return_documents is False, we want to ignore record details (return only IDs) request_data["ignoreRecordDetailsInResponse"] = not return_documents + user_labels = vertex_request_labels_from_litellm_params(litellm_params) + if user_labels: + request_data["userLabels"] = user_labels + return request_data def transform_rerank_response( diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 4baa5774c4..d31e1f6c8f 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -100,6 +100,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 179bd7aeff..6cb7a86bea 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -107,6 +107,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, + extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py b/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py index 18c5ec3d83..696341598e 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional, Union +from typing import Dict, Literal, Optional, Union import httpx @@ -44,6 +44,7 @@ class VertexEmbedding(VertexBase): vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None, gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, + litellm_params: Optional[Dict] = None, ) -> EmbeddingResponse: if aembedding is True: return self.async_embedding( # type: ignore @@ -61,6 +62,7 @@ class VertexEmbedding(VertexBase): vertex_credentials=vertex_credentials, gemini_api_key=gemini_api_key, extra_headers=extra_headers, + litellm_params=litellm_params, ) should_use_v1beta1_features = self.is_using_v1beta1_features( @@ -92,7 +94,10 @@ class VertexEmbedding(VertexBase): headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers) vertex_request: VertexEmbeddingRequest = ( litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request( - input=input, optional_params=optional_params, model=model + input=input, + optional_params=optional_params, + model=model, + litellm_params=litellm_params, ) ) @@ -156,6 +161,7 @@ class VertexEmbedding(VertexBase): gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, encoding=None, + litellm_params: Optional[Dict] = None, ) -> EmbeddingResponse: """ Async embedding implementation @@ -188,7 +194,10 @@ class VertexEmbedding(VertexBase): headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers) vertex_request: VertexEmbeddingRequest = ( litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request( - input=input, optional_params=optional_params, model=model + input=input, + optional_params=optional_params, + model=model, + litellm_params=litellm_params, ) ) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index 132f29987a..24396628db 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -3,6 +3,7 @@ from typing import List, Literal, Optional, Union from pydantic import BaseModel +from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.utils import EmbeddingResponse, Usage from .types import * @@ -100,7 +101,11 @@ class VertexAITextEmbeddingConfig(BaseModel): return optional_params def transform_openai_request_to_vertex_embedding_request( - self, input: Union[list, str], optional_params: dict, model: str + self, + input: Union[list, str], + optional_params: dict, + model: str, + litellm_params: Optional[dict] = None, ) -> VertexEmbeddingRequest: """ Transforms an openai request to a vertex embedding request. @@ -108,16 +113,26 @@ class VertexAITextEmbeddingConfig(BaseModel): # Import here to avoid circular import issues with litellm.__init__ from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig + labels = pop_vertex_request_labels(optional_params, litellm_params) + if model.isdigit(): - return self._transform_openai_request_to_fine_tuned_embedding_request( - input, optional_params, model + vertex_request = ( + self._transform_openai_request_to_fine_tuned_embedding_request( + input, optional_params, model + ) ) + if labels: + vertex_request["labels"] = labels + return vertex_request if VertexBGEConfig.is_bge_model(model): - return VertexBGEConfig.transform_request( + vertex_request = VertexBGEConfig.transform_request( input=input, optional_params=optional_params, model=model ) + if labels: + vertex_request["labels"] = labels + return vertex_request - vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest() + vertex_request = VertexEmbeddingRequest() vertex_text_embedding_input_list: List[TextEmbeddingInput] = [] task_type: Optional[TaskType] = optional_params.get("task_type") title = optional_params.get("title") @@ -133,6 +148,8 @@ class VertexAITextEmbeddingConfig(BaseModel): vertex_request["instances"] = vertex_text_embedding_input_list vertex_request["parameters"] = EmbeddingParameters(**optional_params) + if labels: + vertex_request["labels"] = labels return vertex_request diff --git a/litellm/llms/vertex_ai/vertex_embeddings/types.py b/litellm/llms/vertex_ai/vertex_embeddings/types.py index 317b9c4fb8..bf73f4d193 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/types.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/types.py @@ -3,7 +3,7 @@ Types for Vertex Embeddings Requests """ from enum import Enum -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from typing_extensions import TypedDict @@ -56,6 +56,7 @@ class VertexEmbeddingRequest(TypedDict, total=False): List[TextEmbeddingFineTunedInput], ] parameters: Optional[Union[EmbeddingParameters, TextEmbeddingFineTunedParameters]] + labels: Optional[Dict[str, str]] # Example usage: diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 521dae980d..d64450a121 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -67,7 +67,11 @@ class VoyageRerankConfig(BaseRerankConfig): return api_base def transform_rerank_request( - self, model: str, optional_rerank_params: Dict, headers: Dict + self, + model: str, + optional_rerank_params: Dict, + headers: Dict, + litellm_params: Optional[dict] = None, ) -> Dict: return {"model": model, **optional_rerank_params} diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 4f8e196f25..202760f68a 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -143,6 +143,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to IBM watsonx.ai rerank format diff --git a/litellm/main.py b/litellm/main.py index daa0fb063a..0079bd750c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5311,6 +5311,7 @@ def embedding( # noqa: PLR0915 api_key=api_key, api_base=api_base, client=client, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "oobabooga": response = oobabooga.embedding( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8511d785fb..13a45fd165 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -712,6 +712,7 @@ }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -735,6 +736,7 @@ }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -955,6 +957,7 @@ }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -982,6 +985,7 @@ }, "anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1011,6 +1015,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1040,6 +1045,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1127,6 +1133,7 @@ }, "anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1157,6 +1164,7 @@ }, "global.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1187,6 +1195,7 @@ }, "us.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1277,6 +1286,7 @@ }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "bedrock_converse", @@ -1305,6 +1315,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "bedrock_converse", @@ -1333,6 +1344,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1447,11 +1459,13 @@ }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -5103,6 +5117,38 @@ "/v1/images/edits" ] }, + "azure/gpt-image-2": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true + }, + "azure/gpt-image-2-2026-04-21": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true + }, "azure/low/1024-x-1024/gpt-image-1-mini": { "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", @@ -17889,11 +17935,13 @@ }, "global.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -17950,6 +17998,7 @@ }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -19083,6 +19132,38 @@ "supports_vision": true, "supports_pdf_input": true }, + "gpt-image-2": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true + }, + "gpt-image-2-2026-04-21": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_token": 1e-05, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true + }, "low/1024-x-1024/gpt-image-1.5": { "input_cost_per_image": 0.009, "litellm_provider": "openai", @@ -30052,6 +30133,7 @@ }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -30203,11 +30285,13 @@ }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -30308,6 +30392,7 @@ }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -30335,6 +30420,7 @@ }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index edee50bdfc..c4c9aea6f6 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -21,6 +21,7 @@ import httpx from httpx._types import CookieTypes, QueryParamTypes, RequestFiles import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -390,19 +391,28 @@ def _sync_streaming( ): from litellm.utils import executor + raw_bytes: List[bytes] = [] + flush_scheduled = False try: - raw_bytes: List[bytes] = [] for chunk in response.iter_bytes(): # type: ignore raw_bytes.append(chunk) yield chunk - - executor.submit( - litellm_logging_obj.flush_passthrough_collected_chunks, - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - except Exception as e: - raise e + finally: + if not flush_scheduled and raw_bytes: + flush_scheduled = True + try: + executor.submit( + litellm_logging_obj.flush_passthrough_collected_chunks, + raw_bytes=raw_bytes, + provider_config=provider_config, + ) + except Exception as e: + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush " + "in _sync_streaming; %d buffered chunks dropped: %s", + len(raw_bytes), + e, + ) async def _async_streaming( @@ -411,23 +421,45 @@ async def _async_streaming( provider_config: "BasePassthroughConfig", ): iter_response = await response + try: iter_response.raise_for_status() - raw_bytes: List[bytes] = [] - - async for chunk in iter_response.aiter_bytes(): # type: ignore - raw_bytes.append(chunk) - yield chunk - - asyncio.create_task( - litellm_logging_obj.async_flush_passthrough_collected_chunks( - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - ) except Exception: try: await iter_response.aclose() except Exception: pass raise + + raw_bytes: List[bytes] = [] + flush_scheduled = False + try: + async for chunk in iter_response.aiter_bytes(): # type: ignore + raw_bytes.append(chunk) + yield chunk + except Exception: + try: + await iter_response.aclose() + except Exception: + pass + raise + finally: + # GeneratorExit (raised on client disconnect) is not caught by + # `except Exception`; the finally block ensures partial usage + # still gets flushed for spend tracking. See LIT-2642. + if not flush_scheduled and raw_bytes: + flush_scheduled = True + try: + asyncio.create_task( + litellm_logging_obj.async_flush_passthrough_collected_chunks( + raw_bytes=raw_bytes, + provider_config=provider_config, + ) + ) + except Exception as e: + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush " + "in _async_streaming; %d buffered chunks dropped: %s", + len(raw_bytes), + e, + ) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index d4799e9f20..756b2ed91d 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -117,7 +117,10 @@ class MCPRequestHandler: return b"{}" request.body = mock_body # type: ignore - if ".well-known" in str(request.url): # public routes + # Only OAuth metadata routes registered under /.well-known/ are public. + # Match on request.url.path (path-only, exact prefix) so the substring + # cannot be smuggled via query string, hostname, or a deeper URL segment. + if request.url.path.startswith("/.well-known/"): validated_user_api_key_auth = UserAPIKeyAuth() elif has_explicit_litellm_key: # Explicit x-litellm-api-key provided - always validate normally @@ -126,27 +129,37 @@ class MCPRequestHandler: ) elif oauth2_headers: # No x-litellm-api-key, but Authorization header present. - # Could be a LiteLLM key (backward compat) OR an OAuth2 token - # from an upstream MCP provider (e.g. Atlassian). - # Try LiteLLM auth first; on auth failure, treat as OAuth2 passthrough. + # Could be a LiteLLM key (backward compat) OR an opaque OAuth2 token + # the operator wants forwarded to an upstream OAuth2-mode MCP server. + # Try LiteLLM auth first; on auth failure, only fall back to anonymous + # passthrough when the request actually targets a server whose operator + # configured ``auth_type=oauth2``. For any other server (api_key, + # bearer_token, basic, etc.), a failed LiteLLM auth is a real failure + # and must propagate — otherwise an attacker can exchange any garbage + # bearer for an anonymous session. try: validated_user_api_key_auth = await user_api_key_auth( api_key=litellm_api_key, request=request ) - except HTTPException as e: - if e.status_code in (401, 403): + except (HTTPException, ProxyException) as e: + # HTTPException.status_code is int; ProxyException.code is + # normalized to str in its __init__ but can be ``"None"`` or any + # non-numeric string when the caller didn't supply a numeric + # code, so we compare against both int and str forms rather + # than coercing (``int("None")`` would raise ValueError and + # rewrite the auth error as a 500). + status = e.status_code if isinstance(e, HTTPException) else e.code + if status in ( + 401, + 403, + "401", + "403", + ) and MCPRequestHandler._target_servers_use_oauth2( + path=request.url.path, mcp_servers=mcp_servers + ): verbose_logger.debug( - "MCP OAuth2: Authorization header is not a valid LiteLLM key, " - "treating as OAuth2 token passthrough" - ) - validated_user_api_key_auth = UserAPIKeyAuth() - else: - raise - except ProxyException as e: - if str(e.code) in ("401", "403"): - verbose_logger.debug( - "MCP OAuth2: Authorization header is not a valid LiteLLM key, " - "treating as OAuth2 token passthrough" + "MCP OAuth2: target server is OAuth2-mode, treating " + "Authorization as upstream OAuth2 token passthrough" ) validated_user_api_key_auth = UserAPIKeyAuth() else: @@ -165,6 +178,62 @@ class MCPRequestHandler: dict(headers), ) + @staticmethod + def _extract_target_server_names_from_path(path: str) -> List[str]: + """ + Extract the target MCP server name from the standard MCP transport + URL patterns: ``/mcp/{server_name}[/...]`` and + ``/{server_name}/mcp[/...]``. Returns ``[]`` for any other path so + callers fail closed when the target cannot be resolved. + + REST/admin endpoints, OAuth2 server endpoints + (``/{server_name}/authorize``, ``/token`` etc.), and ``.well-known`` + discovery routes intentionally fall through — those flows do not need + OAuth2 token passthrough. Clients aggregating multiple servers should + use ``x-mcp-servers``, which takes precedence over path parsing. + """ + segments = [s for s in path.split("/") if s] + if len(segments) >= 2 and segments[0] == "mcp": + return [segments[1]] + if len(segments) >= 2 and segments[1] == "mcp": + return [segments[0]] + return [] + + @staticmethod + def _target_servers_use_oauth2(path: str, mcp_servers: Optional[List[str]]) -> bool: + """ + True only when EVERY MCP server the request targets is configured for + ``auth_type == oauth2``. If any target is non-OAuth2 — or if the target + cannot be resolved at all — return False so the caller fails closed. + + Used to gate the "treat Authorization as opaque OAuth2 token" fallback + in :meth:`process_mcp_request` so a failed LiteLLM-auth cannot be + exchanged for an anonymous session against a non-OAuth2 server. + """ + # Inline imports avoid a circular dependency: mcp_server_manager imports + # from this module. + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + + # Use the x-mcp-servers header verbatim when present (including the + # explicitly-empty list, which means "no targets" → fail closed). + # Only fall back to path parsing when the header was absent entirely. + target_names = ( + mcp_servers + if mcp_servers is not None + else MCPRequestHandler._extract_target_server_names_from_path(path) + ) + if not target_names: + return False + + for name in target_names: + server = global_mcp_server_manager.get_mcp_server_by_name(name) + if server is None or server.auth_type != MCPAuth.oauth2: + return False + return True + @staticmethod def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]: """ diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 21abaa3f98..a6f0d145e9 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1,4 +1,5 @@ import base64 +import binascii import json from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Set, Union, cast @@ -498,6 +499,82 @@ async def rotate_mcp_server_credentials_master_key( ) +def _decode_user_credential(stored: str) -> Optional[str]: + """Read back a value persisted in ``LiteLLM_MCPUserCredentials.credential_b64``. + + Tries nacl decryption first (current write format). Falls back to a + plain ``urlsafe_b64decode`` for rows persisted by older code that wrote + the credential without encryption. Returns ``None`` when neither path + yields a valid string. + """ + decrypted = decrypt_value_helper( + value=stored, + key="mcp_user_credential", + exception_type="debug", + return_original_value=False, + ) + if decrypted is not None: + return decrypted + try: + return base64.urlsafe_b64decode(stored).decode() + except (binascii.Error, UnicodeDecodeError, ValueError, TypeError): + return None + + +def _decode_oauth_payload(stored: str) -> Optional[Dict[str, Any]]: + """Return the OAuth2 payload dict if ``stored`` holds one, else ``None``. + + A row is considered an OAuth2 credential iff its decoded value parses as + a JSON object with ``"type": "oauth2"``. Plain BYOK credentials (which + share the same column) decode to a non-JSON string and return ``None``. + """ + decoded = _decode_user_credential(stored) + if decoded is None: + return None + try: + parsed = json.loads(decoded) + except (ValueError, TypeError): + return None + if isinstance(parsed, dict) and parsed.get("type") == "oauth2": + return parsed + return None + + +async def rotate_mcp_user_credentials_master_key( + prisma_client: PrismaClient, new_master_key: str +): + """Re-encrypt every ``LiteLLM_MCPUserCredentials`` row with ``new_master_key``. + + Reads each ``credential_b64`` with the current salt key (falling back to + legacy plain base64 for unmigrated rows) and writes it back encrypted + under the new master key. Rows that are unreadable under both paths + are logged and skipped so one corrupt row does not abort the rotation. + """ + rows = await prisma_client.db.litellm_mcpusercredentials.find_many() + for row in rows: + plaintext = _decode_user_credential(row.credential_b64) + if plaintext is None: + verbose_proxy_logger.warning( + "rotate_mcp_user_credentials_master_key: could not decode " + "credential for user_id=%s server_id=%s, skipping", + row.user_id, + row.server_id, + ) + continue + re_encrypted = encrypt_value_helper( + plaintext, new_encryption_key=new_master_key + ) + await prisma_client.db.litellm_mcpusercredentials.update( + where={ + "user_id_server_id": { + "user_id": row.user_id, + "server_id": row.server_id, + } + }, + data={"credential_b64": re_encrypted}, + ) + + async def store_user_credential( prisma_client: PrismaClient, user_id: str, @@ -506,7 +583,7 @@ async def store_user_credential( ) -> None: """Store a user credential for a BYOK MCP server.""" - encoded = base64.urlsafe_b64encode(credential.encode()).decode() + encoded = encrypt_value_helper(credential) await prisma_client.db.litellm_mcpusercredentials.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ @@ -532,16 +609,7 @@ async def get_user_credential( ) if row is None: return None - try: - return base64.urlsafe_b64decode(row.credential_b64).decode() - except Exception: - # Fall back to nacl decryption for credentials stored by older code - return decrypt_value_helper( - value=row.credential_b64, - key="byok_credential", - exception_type="debug", - return_original_value=False, - ) + return _decode_user_credential(row.credential_b64) async def has_user_credential( @@ -582,7 +650,7 @@ async def store_user_oauth_credential( ) -> None: """Persist an OAuth2 access token for a user+server pair. - The payload is JSON-serialised and stored base64-encoded in the same + The payload is JSON-serialised and stored encrypted in the same ``credential_b64`` column used by BYOK. A ``"type": "oauth2"`` key differentiates it from plain BYOK API keys. """ @@ -606,29 +674,27 @@ async def store_user_oauth_credential( payload["scopes"] = scopes # Guard against silently overwriting a BYOK credential with an OAuth token. - # BYOK credentials lack a "type" field (or use a non-"oauth2" type). # Skip the guard when the caller knows the row is already an OAuth2 credential # (e.g. during token refresh), saving an extra DB round-trip. if not skip_byok_guard: existing = await prisma_client.db.litellm_mcpusercredentials.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) - if existing is not None: - _byok_error = ValueError( - f"A non-OAuth2 credential already exists for user {user_id} " - f"and server {server_id}. Refusing to overwrite." + if ( + existing is not None + and _decode_oauth_payload(existing.credential_b64) is None + ): + # Existing row is either a BYOK secret or an OAuth2 row that no + # longer decrypts (e.g. after a salt-key rotation). In either + # case, refuse to overwrite — the caller would clobber data + # that may still be recoverable. + raise ValueError( + f"Existing credential for user {user_id} and server " + f"{server_id} could not be verified as an OAuth2 token. " + f"Refusing to overwrite." ) - try: - raw = json.loads( - base64.urlsafe_b64decode(existing.credential_b64).decode() - ) - except Exception: - # Credential is not base64+JSON — it's a plain-text BYOK key. - raise _byok_error - if raw.get("type") != "oauth2": - raise _byok_error - encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + encoded = encrypt_value_helper(json.dumps(payload)) await prisma_client.db.litellm_mcpusercredentials.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ @@ -672,15 +738,7 @@ async def get_user_oauth_credential( ) if row is None: return None - try: - decoded = base64.urlsafe_b64decode(row.credential_b64).decode() - parsed = json.loads(decoded) - if isinstance(parsed, dict) and parsed.get("type") == "oauth2": - return parsed - # Row exists but is a BYOK (plain string), not an OAuth token - return None - except Exception: - return None + return _decode_oauth_payload(row.credential_b64) async def list_user_oauth_credentials( @@ -694,14 +752,11 @@ async def list_user_oauth_credentials( ) results: List[Dict[str, Any]] = [] for row in rows: - try: - decoded = base64.urlsafe_b64decode(row.credential_b64).decode() - parsed = json.loads(decoded) - if isinstance(parsed, dict) and parsed.get("type") == "oauth2": - parsed["server_id"] = row.server_id - results.append(parsed) - except Exception: - pass # Skip non-OAuth rows (BYOK plain strings) + payload = _decode_oauth_payload(row.credential_b64) + if payload is None: + continue + payload["server_id"] = row.server_id + results.append(payload) return results diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 13b19aa2d6..cebd224a1a 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -131,6 +131,22 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data +def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str: + """Return a loopback client redirect URI from OAuth state.""" + redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url") + if not redirect_uri or not isinstance(redirect_uri, str): + raise HTTPException(status_code=400, detail="Invalid redirect URI") + validate_loopback_redirect_uri(redirect_uri) + return redirect_uri + + +def _append_query_params(url: str, params: Dict[str, str]) -> str: + parsed = urlparse(url) + query_params = parse_qsl(parsed.query, keep_blank_values=True) + query_params.extend(params.items()) + return urlunparse(parsed._replace(query=urlencode(query_params))) + + def _resolve_oauth2_server_for_root_endpoints( client_ip: Optional[str] = None, ) -> Optional[MCPServer]: @@ -568,7 +584,7 @@ async def authorize( else None ) if mcp_server is None and mcp_server_name is None: - mcp_server = _resolve_oauth2_server_for_root_endpoints() + mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") # Use server's stored client_id when caller doesn't supply one. @@ -630,7 +646,7 @@ async def token_endpoint( lookup_name, client_ip=client_ip ) if mcp_server is None and mcp_server_name is None: - mcp_server = _resolve_oauth2_server_for_root_endpoints() + mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") return await exchange_token_with_server( @@ -651,7 +667,6 @@ async def token_endpoint( async def callback(code: str, state: str): try: state_data = decode_state_hash(state) - base_url = state_data["base_url"] original_state = state_data["original_state"] # Re-validate loopback at the sink. /authorize rejects non-loopback @@ -659,10 +674,10 @@ async def callback(code: str, state: str): # minted before that check was added have no expiry and remain # valid indefinitely. Validating here blocks the open-redirect + # code-theft primitive even for pre-fix states. - validate_loopback_redirect_uri(base_url) + redirect_uri = _get_validated_client_redirect_uri(state_data) params = {"code": code, "state": original_state} - complete_returned_url = f"{base_url}?{urlencode(params)}" + complete_returned_url = _append_query_params(redirect_uri, params) return RedirectResponse(url=complete_returned_url, status_code=302) except HTTPException: @@ -719,16 +734,16 @@ def _build_oauth_protected_resource_response( ) request_base_url = get_request_base_url(request) + client_ip = IPAddressUtils.get_mcp_client_ip(request) # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: mcp_server_name = resolved.server_name or resolved.name mcp_server: Optional[MCPServer] = None if mcp_server_name: - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) @@ -835,10 +850,11 @@ def _build_oauth_authorization_server_response( ) request_base_url = get_request_base_url(request) + client_ip = IPAddressUtils.get_mcp_client_ip(request) # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: mcp_server_name = resolved.server_name or resolved.name @@ -855,7 +871,6 @@ def _build_oauth_authorization_server_response( mcp_server: Optional[MCPServer] = None if mcp_server_name: - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) @@ -1007,8 +1022,9 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non "client_secret": "dummy", "redirect_uris": [f"{request_base_url}/callback"], } + client_ip = IPAddressUtils.get_mcp_client_ip(request) if not mcp_server_name: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: return await register_client_with_server( request=request, @@ -1021,7 +1037,6 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non ) return dummy_return - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 251f271903..f96350500d 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -41,6 +41,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( @@ -50,8 +51,11 @@ from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mc from litellm.proxy._experimental.mcp_server.utils import ( MCP_TOOL_PREFIX_SEPARATOR, add_server_prefix_to_name, + compute_short_server_prefix, get_server_prefix, + is_short_mcp_tool_prefix_enabled, is_tool_name_prefixed, + iter_known_server_prefixes, merge_mcp_headers, normalize_server_name, split_server_prefix_from_name, @@ -106,6 +110,12 @@ if not _separator_probe.is_valid: SEP_986_URL, ) +_AZURE_ENTRA_HOSTS = { + "login.microsoftonline.com", # Global + "login.microsoftonline.us", # US Government + "login.chinacloudapi.cn", # China +} + def _warn_on_server_name_fields( *, @@ -364,6 +374,7 @@ class MCPServerManager: aws_session_name=server_config.get("aws_session_name", None), instructions=server_config.get("instructions", None), ) + self._assign_unique_short_prefix(new_server) self.config_mcp_servers[server_id] = new_server # Check if this is an OpenAPI-based server @@ -726,6 +737,7 @@ class MCPServerManager: try: if mcp_server.server_id not in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) + self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) verbose_logger.debug(f"Added MCP Server: {new_server.name}") @@ -738,6 +750,12 @@ class MCPServerManager: try: if mcp_server.server_id in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) + # Carry the previously-resolved short prefix across so the + # tool names stay stable for clients holding cached lists. + existing_prefix = self.registry[mcp_server.server_id].short_prefix + if existing_prefix and not new_server.short_prefix: + new_server.short_prefix = existing_prefix + self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) verbose_logger.debug(f"Updated MCP Server: {new_server.name}") @@ -1236,7 +1254,11 @@ class MCPServerManager: ## HANDLE OPENAPI TOOLS if server.spec_path: - _tools = global_mcp_tool_registry.list_tools(tool_prefix=server.name) + # OpenAPI tools were stored in the registry under the prefix + # active at registration time — fetch by that same prefix. + _tools = global_mcp_tool_registry.list_tools( + tool_prefix=get_server_prefix(server) + ) tools = global_mcp_tool_registry.convert_tools_to_mcp_sdk_tool_type( _tools ) @@ -1478,6 +1500,47 @@ class MCPServerManager: ) return await client.get_prompt(get_prompt_request_params) + @staticmethod + def _is_same_authority_metadata_url(url: str, server_url: str) -> bool: + """ + Whether ``url`` shares scheme, host, and port with ``server_url``. + + Same-authority metadata URLs are produced by our well-known discovery + construction and by resource servers that publish protected-resource + metadata on the resource origin. These must keep working for + administrator-configured internal MCP servers, so they are fetched + directly. Cross-origin URLs are fetched through ``async_safe_get``. + """ + try: + target = urlparse(url) + base = urlparse(server_url) + except Exception: + return False + + if target.scheme not in ("http", "https") or not target.hostname: + return False + + target_port = target.port or (443 if target.scheme == "https" else 80) + base_port = base.port or (443 if base.scheme == "https" else 80) + return ( + base.scheme == target.scheme + and (base.hostname or "").lower() == target.hostname.lower() + and base_port == target_port + ) + + async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> Any: + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.MCP, + params={"timeout": MCP_METADATA_TIMEOUT}, + ) + if self._is_same_authority_metadata_url(url, server_url): + # Same-authority URLs may point at administrator-configured + # internal MCP servers. Do not run them through user URL + # validation, but also do not follow redirects because the + # redirect target would not inherit the same-authority guarantee. + return await client.get(url, follow_redirects=False) + return await async_safe_get(client, url) + async def _descovery_metadata( self, server_url: str, @@ -1488,11 +1551,28 @@ class MCPServerManager: client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) response = await client.get(server_url) response.raise_for_status() - verbose_logger.warning( - "MCP OAuth discovery unexpectedly succeeded for %s; server did not challenge", - server_url, + ( + authorization_servers, + resource_scopes, + ) = await self._attempt_well_known_discovery(server_url) + metadata = await self._fetch_authorization_server_metadata( + authorization_servers, server_url ) - raise RuntimeError("OAuth discovery must not succeed without a challenge") + if ( + metadata is None + and not resource_scopes + and authorization_servers + and response.status_code == 200 + ): + verbose_logger.warning( + "MCP OAuth discovery for %s received 200 OK without RFC 9728 challenge and no discoverable authorization metadata.", + server_url, + ) + if metadata is None and resource_scopes: + return MCPOAuthMetadata(scopes=resource_scopes) + if metadata is not None and resource_scopes: + metadata.scopes = resource_scopes + return metadata except HTTPStatusError as exc: verbose_logger.debug( "MCP OAuth discovery for %s received status error: %s", @@ -1510,14 +1590,14 @@ class MCPServerManager: header_value ) - authorization_servers: List[str] = [] - resource_scopes: Optional[List[str]] = None + authorization_servers = [] + resource_scopes = None if resource_metadata_url: ( authorization_servers, resource_scopes, ) = await self._fetch_oauth_metadata_from_resource( - resource_metadata_url + resource_metadata_url, server_url ) else: ( @@ -1538,7 +1618,7 @@ class MCPServerManager: if authorization_servers: metadata = await self._fetch_authorization_server_metadata( - authorization_servers + authorization_servers, server_url ) preferred_scopes = scopes or resource_scopes @@ -1578,19 +1658,26 @@ class MCPServerManager: return resource_metadata_url, scopes async def _fetch_oauth_metadata_from_resource( - self, resource_metadata_url: str + self, resource_metadata_url: str, server_url: str ) -> Tuple[List[str], Optional[List[str]]]: if not resource_metadata_url: return [], None try: - client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.MCP, - params={"timeout": MCP_METADATA_TIMEOUT}, + response = await self._fetch_oauth_discovery_url( + resource_metadata_url, server_url ) - response = await client.get(resource_metadata_url) response.raise_for_status() data = response.json() + except SSRFError as exc: + verbose_logger.warning( + "MCP OAuth discovery: refusing to fetch resource metadata from %s " + "(rejected by SSRF guard for server %s): %s", + resource_metadata_url, + server_url, + exc, + ) + return [], None except Exception as exc: # pragma: no cover - network issues verbose_logger.debug( "Failed to fetch MCP OAuth metadata from %s: %s", @@ -1639,23 +1726,25 @@ class MCPServerManager: ( authorization_servers, scopes, - ) = await self._fetch_oauth_metadata_from_resource(url) + ) = await self._fetch_oauth_metadata_from_resource(url, server_url) if authorization_servers: return authorization_servers, scopes return [], None async def _fetch_authorization_server_metadata( - self, authorization_servers: List[str] + self, authorization_servers: List[str], server_url: str ) -> Optional[MCPOAuthMetadata]: for issuer in authorization_servers: - metadata = await self._fetch_single_authorization_server_metadata(issuer) + metadata = await self._fetch_single_authorization_server_metadata( + issuer, server_url + ) if metadata is not None: return metadata return None async def _fetch_single_authorization_server_metadata( - self, issuer_url: str + self, issuer_url: str, server_url: str ) -> Optional[MCPOAuthMetadata]: try: parsed = urlparse(issuer_url) @@ -1674,19 +1763,27 @@ class MCPServerManager: f"{base}/.well-known/oauth-authorization-server/{path}" ) candidate_urls.append(f"{base}/.well-known/openid-configuration/{path}") + candidate_urls.append( + f"{issuer_url.rstrip('/')}/.well-known/openid-configuration" + ) candidate_urls.append(f"{base}/.well-known/oauth-authorization-server") candidate_urls.append(f"{base}/.well-known/openid-configuration") candidate_urls.append(issuer_url.rstrip("/")) for url in candidate_urls: try: - client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.MCP, - params={"timeout": MCP_METADATA_TIMEOUT}, - ) - response = await client.get(url) + response = await self._fetch_oauth_discovery_url(url, server_url) response.raise_for_status() data = response.json() + except SSRFError as exc: + verbose_logger.warning( + "MCP OAuth discovery: refusing to fetch authorization-server " + "metadata from %s (rejected by SSRF guard for server %s): %s", + url, + server_url, + exc, + ) + continue except Exception as exc: # pragma: no cover - network issues verbose_logger.debug( "Failed to fetch authorization metadata from %s: %s", @@ -1713,7 +1810,28 @@ class MCPServerManager: ): return metadata - return None + return self._build_azure_authorization_server_metadata(parsed) + + @staticmethod + def _build_azure_authorization_server_metadata( + parsed_issuer_url: Any, + ) -> Optional[MCPOAuthMetadata]: + path_parts = [ + part for part in (parsed_issuer_url.path or "").split("/") if part + ] + if ( + parsed_issuer_url.netloc not in _AZURE_ENTRA_HOSTS + or len(path_parts) != 2 + or path_parts[1] != "v2.0" + ): + return None + + tenant = path_parts[0] + base = f"{parsed_issuer_url.scheme}://{parsed_issuer_url.netloc}/{tenant}" + return MCPOAuthMetadata( + authorization_url=f"{base}/oauth2/v2.0/authorize", + token_url=f"{base}/oauth2/v2.0/token", + ) @staticmethod def _decrypt_credential_field( @@ -1810,6 +1928,63 @@ class MCPServerManager: verbose_logger.warning(f"Error listing tools from {server_name}: {str(e)}") return [] + _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024 + + def _assign_unique_short_prefix(self, server: MCPServer) -> None: + """Resolve and cache a collision-free short tool prefix on ``server``. + + Called at registration time for every MCP server entering the + registry. Mutates ``server.short_prefix`` in place. No-ops when + ``LITELLM_USE_SHORT_MCP_TOOL_PREFIX`` is disabled, when the server + has no ``server_id`` (synthetic temp-server objects), or when a + prefix is already cached. + + Collision strategy: take the natural hash; if it's already used by + a *different* server in the combined registry, rehash with an + incrementing attempt counter until we find an unused slot. The + attempt counter is folded into the hash so the resulting prefix is + still deterministic for a given (server_id, set-of-other-server-ids) + pair within one process. + """ + if not is_short_mcp_tool_prefix_enabled(): + return + if server.short_prefix: + return + if not server.server_id: + return + + used: Dict[str, str] = {} + for other in self.get_registry().values(): + if other.server_id == server.server_id: + continue + if other.short_prefix: + used[other.short_prefix] = other.server_id + + for attempt in range(self._SHORT_PREFIX_MAX_REHASH_ATTEMPTS): + candidate = compute_short_server_prefix(server.server_id, attempt=attempt) + if candidate not in used: + server.short_prefix = candidate + if attempt > 0: + verbose_logger.info( + "MCP short-prefix collision resolved for server %s: " + "natural hash collided with %s, using rehashed prefix " + "%s (attempt=%d).", + server.server_id, + used.get( + compute_short_server_prefix(server.server_id, attempt=0), + "", + ), + candidate, + attempt, + ) + return + + raise RuntimeError( + f"Unable to assign a unique short MCP tool prefix for server " + f"{server.server_id} after {self._SHORT_PREFIX_MAX_REHASH_ATTEMPTS} " + "attempts; the 3-character prefix space is too crowded." + ) + def _create_prefixed_tools( self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True ) -> List[MCPTool]: @@ -1838,9 +2013,13 @@ class MCPServerManager: tool_copy.name = name_to_use prefixed_tools.append(tool_copy) - # Update tool to server mapping for resolution (support both forms) + # Register every known prefix form (alias, server_name, server_id, + # short ID) so call_tool can resolve regardless of which form a + # caller / cached client is using. self.tool_name_to_mcp_server_name_mapping[original_name] = prefix - self.tool_name_to_mcp_server_name_mapping[prefixed_name] = prefix + for known_prefix in iter_known_server_prefixes(server): + qualified = add_server_prefix_to_name(original_name, known_prefix) + self.tool_name_to_mcp_server_name_mapping[qualified] = prefix verbose_logger.info( f"Successfully fetched {len(prefixed_tools)} tools from server {server.name}" @@ -2601,37 +2780,43 @@ class MCPServerManager: Returns: MCPServer if found, None otherwise """ + registry_servers = list(self.get_registry().values()) + + # Build prefix → server lookup covering every known form a tool name + # may take (alias / server_name / server_id / short ID). This is what + # makes the short-prefix mode work without breaking historical names. + prefix_to_server: Dict[str, MCPServer] = {} + for server in registry_servers: + for known_prefix in iter_known_server_prefixes(server): + normalised = normalize_server_name(known_prefix) + prefix_to_server.setdefault(normalised, server) + # First try with the original tool name if tool_name in self.tool_name_to_mcp_server_name_mapping: server_name = self.tool_name_to_mcp_server_name_mapping[tool_name] - for server in self.get_registry().values(): - if normalize_server_name(server.name) == normalize_server_name( - server_name - ): + normalised_lookup = normalize_server_name(server_name) + if normalised_lookup in prefix_to_server: + return prefix_to_server[normalised_lookup] + for server in registry_servers: + if normalize_server_name(server.name) == normalised_lookup: return server - # If not found and tool name is prefixed, try extracting server name from prefix - known_prefixes = { - normalize_server_name(get_server_prefix(s)) - for s in self.get_registry().values() - if get_server_prefix(s) - } - if is_tool_name_prefixed(tool_name, known_server_prefixes=known_prefixes): + # If not found and tool name is prefixed, extract the prefix and + # match against any known form. + if is_tool_name_prefixed( + tool_name, known_server_prefixes=set(prefix_to_server.keys()) + ): ( original_tool_name, server_name_from_prefix, ) = split_server_prefix_from_name(tool_name) - if original_tool_name in self.tool_name_to_mcp_server_name_mapping: - for server in self.get_registry().values(): - if server.server_name is None: - if normalize_server_name(server.name) == normalize_server_name( - server_name_from_prefix - ): - return server - elif normalize_server_name( - server.server_name - ) == normalize_server_name(server_name_from_prefix): - return server + normalised_prefix = normalize_server_name(server_name_from_prefix) + matched_server = prefix_to_server.get(normalised_prefix) + if matched_server is not None and ( + original_tool_name in self.tool_name_to_mcp_server_name_mapping + or tool_name in self.tool_name_to_mcp_server_name_mapping + ): + return matched_server return None @@ -2666,6 +2851,9 @@ class MCPServerManager: previous_registry = self.registry new_registry: Dict[str, MCPServer] = {} + # Stage one: build every server. Stage two assigns short prefixes + # against the *full* set so dedup is deterministic regardless of + # iteration order. for server in db_mcp_servers: existing_server = previous_registry.get(server.server_id) @@ -2689,10 +2877,21 @@ class MCPServerManager: f"Building server from DB: {server.server_id} ({server.server_name})" ) new_server = await self.build_mcp_server_from_table(server) + # Carry the cached short_prefix from the previous registry entry + # (if any) so the prefix is stable across reloads. + if existing_server is not None and existing_server.short_prefix: + new_server.short_prefix = existing_server.short_prefix new_registry[server.server_id] = new_server - await self._maybe_register_openapi_tools(new_server) + # Swap in the new registry first so _assign_unique_short_prefix + # sees the complete set when checking for collisions. self.registry = new_registry + for new_server in new_registry.values(): + self._assign_unique_short_prefix(new_server) + # Register OpenAPI tools *after* the final short prefix is assigned + # so the tools are stored in the global registry under the same + # prefix that lookups will use. + await self._maybe_register_openapi_tools(new_server) verbose_logger.debug( "MCP registry refreshed (%s servers in registry)", len(new_registry) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index c2e998f01e..ae6055217b 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -49,6 +49,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_VERSION, add_server_prefix_to_name, get_server_prefix, + iter_known_server_prefixes, ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils @@ -711,13 +712,7 @@ if MCP_AVAILABLE: for server in allowed_mcp_servers: if server: match_list = [ - s.lower() - for s in [ - server.alias, - server.server_name, - server.server_id, - ] - if s is not None + s.lower() for s in iter_known_server_prefixes(server) if s ] if server_or_group.lower() in match_list: @@ -2031,11 +2026,13 @@ if MCP_AVAILABLE: # Remove prefix from tool name for logging and processing original_tool_name, server_name = split_server_prefix_from_name(name) - # If tool name is unprefixed, resolve its server so we can enforce permissions - if not server_name: - mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) - if mcp_server: - server_name = mcp_server.name + # Resolve the actual MCP server up-front so the permission check uses + # the canonical server.name even when the tool name is prefixed with a + # short ID (LITELLM_USE_SHORT_MCP_TOOL_PREFIX) that doesn't match the + # server's display name directly. + mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + if mcp_server is not None: + server_name = mcp_server.name # Only enforce server-level permissions when we can resolve a server if server_name: diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 146ba10bb7..df5705c342 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -2,10 +2,11 @@ MCP Server Utilities """ -from typing import Any, Dict, Mapping, Optional, Tuple +from typing import Any, Dict, Iterator, Mapping, Optional, Tuple -import os +import hashlib import importlib +import os # Constants LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" @@ -14,6 +15,89 @@ LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-") MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}" +# --------------------------------------------------------------------------- +# Short-ID tool prefix (opt-in) +# --------------------------------------------------------------------------- +# When LITELLM_USE_SHORT_MCP_TOOL_PREFIX is truthy the prefix attached to MCP +# tool / prompt / resource / resource-template names switches from the +# (potentially long) human-readable server name to a deterministic three +# character ID derived from the server's ``server_id``. +# +# Why three characters? +# * The first character is restricted to 52 alphabetic characters +# ([A-Za-z]) and the remaining two characters use the full base62 +# alphabet ([0-9A-Za-z]). That guarantees the prefix never starts +# with a digit so it remains a valid identifier for every model API +# (some providers historically required a leading alphabetic char). +# * 52 * 62 * 62 = 199_888 distinct IDs. The chance of a real local +# tool name happening to begin with the exact prefix LiteLLM assigned +# to a given MCP server is negligible in practice. +# * The IDs are short enough that prefixed tool names stay well under +# the 60-character upper bound enforced by some model APIs (Anthropic +# etc.) even for long upstream tool names. +# * The mapping is deterministic (SHA-256 of ``server_id`` → three +# characters drawn from the alphabets above), so the prefix is stable +# across processes, workers and restarts without any persistence +# layer. Two servers with different ``server_id`` values can in +# principle hash to the same three chars; that natural-hash collision +# IS a routing-correctness issue (the second registrant would otherwise +# have its tools misrouted to the first), so registration goes through +# ``MCPServerManager._assign_unique_short_prefix`` which rehashes with +# a deterministic attempt counter until it finds an unused prefix and +# caches the result on ``MCPServer.short_prefix``. A collision is +# logged at INFO when it happens. +# +# This flag is intentionally opt-in for the first release so customers can +# migrate. It will become the default in a future release. +SHORT_MCP_TOOL_PREFIX_LENGTH = 3 +_BASE62_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +# Subset of _BASE62_ALPHABET used for the *first* character only, to +# guarantee the prefix never starts with a digit. +_BASE52_ALPHA_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + +def is_short_mcp_tool_prefix_enabled() -> bool: + """Return True when the short-ID tool prefix mode is enabled. + + Read at call time (not import time) so tests and runtime config changes + take effect without reimporting the module. + """ + raw = os.environ.get("LITELLM_USE_SHORT_MCP_TOOL_PREFIX", "") + return raw.strip().lower() in ("1", "true", "yes", "on") + + +def compute_short_server_prefix(server_id: str, attempt: int = 0) -> str: + """Derive the deterministic three-character prefix for a server. + + Uses SHA-256 of ``f"{server_id}#{attempt}"`` and folds the first eight + bytes into a fixed-length string whose first character is drawn from + ``_BASE52_ALPHA_ALPHABET`` (so the prefix never starts with a digit) + and whose remaining characters are drawn from the full base62 + alphabet. Pass ``attempt > 0`` to rehash to a different prefix when + the natural hash collides with a prefix already assigned to another + server (see ``MCPServerManager._assign_unique_short_prefix``). An + empty ``server_id`` raises ``ValueError`` — short prefixes require a + stable identifier to be deterministic. + """ + if not server_id: + raise ValueError("compute_short_server_prefix requires a non-empty server_id") + + seed = server_id if attempt == 0 else f"{server_id}#{attempt}" + digest = hashlib.sha256(seed.encode("utf-8")).digest() + value = int.from_bytes(digest[:8], "big") + + # Build chars from least-significant to most-significant; we reverse + # at the end so the first emitted char comes from the high-order + # bits of the digest (which is the position we constrain to be + # alphabetic). + chars = [] + for position in range(SHORT_MCP_TOOL_PREFIX_LENGTH): + is_first_char = position == SHORT_MCP_TOOL_PREFIX_LENGTH - 1 + alphabet = _BASE52_ALPHA_ALPHABET if is_first_char else _BASE62_ALPHABET + value, idx = divmod(value, len(alphabet)) + chars.append(alphabet[idx]) + return "".join(reversed(chars)) + def is_mcp_available() -> bool: """ @@ -82,7 +166,25 @@ def add_server_prefix_to_name(name: str, server_name: str) -> str: def get_server_prefix(server: Any) -> str: - """Return the prefix for a server: alias if present, else server_name, else server_id""" + """Return the prefix for a server. + + When the short-prefix mode is enabled (``LITELLM_USE_SHORT_MCP_TOOL_PREFIX``) + a three-character base62 ID is returned. We prefer the cached + ``server.short_prefix`` value when set — that field is populated at + registration time by ``MCPServerManager._assign_unique_short_prefix`` + and resolves natural-hash collisions deterministically — and only fall + back to the natural hash for ad-hoc / temp-server objects without a + cached value. In default mode the historical behaviour is preserved: + alias if present, else server_name, else server_id. + """ + if is_short_mcp_tool_prefix_enabled(): + cached = getattr(server, "short_prefix", None) + if cached: + return cached + server_id = getattr(server, "server_id", None) + if server_id: + return compute_short_server_prefix(server_id) + if hasattr(server, "alias") and server.alias: return server.alias if hasattr(server, "server_name") and server.server_name: @@ -92,6 +194,36 @@ def get_server_prefix(server: Any) -> str: return "" +def iter_known_server_prefixes(server: Any) -> Iterator[str]: + """Yield every prefix form that may appear in tool names for ``server``. + + Always includes the *current* prefix returned by ``get_server_prefix``. + Additionally yields the historical (alias / server_name / server_id) and + short-ID forms so the routing layer can resolve tool names regardless of + which prefix mode was active when the client first observed them. + """ + seen = set() + + def _emit(value: Optional[str]) -> Iterator[str]: + if value and value not in seen: + seen.add(value) + yield value + + yield from _emit(get_server_prefix(server)) + yield from _emit(getattr(server, "short_prefix", None)) + + server_id = getattr(server, "server_id", None) + if server_id: + try: + yield from _emit(compute_short_server_prefix(server_id)) + except ValueError: + pass + + yield from _emit(getattr(server, "alias", None)) + yield from _emit(getattr(server, "server_name", None)) + yield from _emit(server_id) + + def split_server_prefix_from_name(prefixed_name: str) -> Tuple[str, str]: """Return the unprefixed name plus the server name used as prefix.""" if MCP_TOOL_PREFIX_SEPARATOR in prefixed_name: diff --git a/litellm/proxy/_experimental/out/chat.html b/litellm/proxy/_experimental/out/chat.html deleted file mode 100644 index 6ca94ee94f..0000000000 --- a/litellm/proxy/_experimental/out/chat.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/chat.txt b/litellm/proxy/_experimental/out/chat.txt deleted file mode 100644 index 8ee24df074..0000000000 --- a/litellm/proxy/_experimental/out/chat.txt +++ /dev/null @@ -1,22 +0,0 @@ -1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -7:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"] -a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] -b:"$Sreact.suspense" -d:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] -f:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] -11:I[168027,[],"default"] -:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"] -:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -0:{"P":null,"b":"zxkD4-EPlgfKHDTw8O869","c":["","chat"],"q":"","i":false,"f":[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true,"nonce":"$undefined"}]],["$","$La",null,{"children":["$","$b",null,{"name":"Next.MetadataOutlet","children":"$@c"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$Ld",null,{"children":"$Le"}],["$","div",null,{"hidden":true,"children":["$","$Lf",null,{"children":["$","$b",null,{"name":"Next.Metadata","children":"$L10"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$11",[]],"S":true} -8:{} -9:"$0:f:0:1:1:children:1:children:0:props:children:0:props:serverProvidedParams:params" -e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] -12:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -c:null -10:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L12","4",{}]] diff --git a/litellm/proxy/_experimental/out/chat/__next._full.txt b/litellm/proxy/_experimental/out/chat/__next._full.txt deleted file mode 100644 index 8ee24df074..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next._full.txt +++ /dev/null @@ -1,22 +0,0 @@ -1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -7:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"] -a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] -b:"$Sreact.suspense" -d:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] -f:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] -11:I[168027,[],"default"] -:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"] -:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -0:{"P":null,"b":"zxkD4-EPlgfKHDTw8O869","c":["","chat"],"q":"","i":false,"f":[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true,"nonce":"$undefined"}]],["$","$La",null,{"children":["$","$b",null,{"name":"Next.MetadataOutlet","children":"$@c"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$Ld",null,{"children":"$Le"}],["$","div",null,{"hidden":true,"children":["$","$Lf",null,{"children":["$","$b",null,{"name":"Next.Metadata","children":"$L10"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$11",[]],"S":true} -8:{} -9:"$0:f:0:1:1:children:1:children:0:props:children:0:props:serverProvidedParams:params" -e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] -12:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -c:null -10:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L12","4",{}]] diff --git a/litellm/proxy/_experimental/out/chat/__next._head.txt b/litellm/proxy/_experimental/out/chat/__next._head.txt deleted file mode 100644 index 635c8e398b..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next._head.txt +++ /dev/null @@ -1,6 +0,0 @@ -1:"$Sreact.fragment" -2:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] -3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] -4:"$Sreact.suspense" -5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/chat/__next._index.txt b/litellm/proxy/_experimental/out/chat/__next._index.txt deleted file mode 100644 index d026a48036..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next._index.txt +++ /dev/null @@ -1,8 +0,0 @@ -1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"] -0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/chat/__next._tree.txt b/litellm/proxy/_experimental/out/chat/__next._tree.txt deleted file mode 100644 index afb8782562..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next._tree.txt +++ /dev/null @@ -1,4 +0,0 @@ -:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"] -:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -0:{"buildId":"zxkD4-EPlgfKHDTw8O869","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"chat","paramType":null,"paramKey":"chat","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300} diff --git a/litellm/proxy/_experimental/out/chat/__next.chat.__PAGE__.txt b/litellm/proxy/_experimental/out/chat/__next.chat.__PAGE__.txt deleted file mode 100644 index 5d19f68046..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next.chat.__PAGE__.txt +++ /dev/null @@ -1,9 +0,0 @@ -1:"$Sreact.fragment" -2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -3:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"] -6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] -7:"$Sreact.suspense" -0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} -4:{} -5:"$0:rsc:props:children:0:props:serverProvidedParams:params" -8:null diff --git a/litellm/proxy/_experimental/out/chat/__next.chat.txt b/litellm/proxy/_experimental/out/chat/__next.chat.txt deleted file mode 100644 index 12f813af35..0000000000 --- a/litellm/proxy/_experimental/out/chat/__next.chat.txt +++ /dev/null @@ -1,4 +0,0 @@ -1:"$Sreact.fragment" -2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py new file mode 100644 index 0000000000..d58bfbbdb5 --- /dev/null +++ b/litellm/proxy/_lazy_features.py @@ -0,0 +1,432 @@ +""" +Lazy registration for optional feature routers. Each LAZY_FEATURES entry +imports its module only on the first request matching its path prefix, +saving ~700 MB at idle for deployments that don't use these features. +First hit pays the import cost (1-3 s for heavy modules); /openapi.json +omits each feature's routes until the feature is warmed. +""" + +import asyncio +import importlib +import sys +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Dict, Tuple + +from starlette.types import Receive, Scope, Send + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from fastapi import APIRouter, FastAPI + + +def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + app.include_router(getattr(module, attr_name)) + + return _register + + +def _mount_app( + prefix: str, attr_name: str = "app" +) -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + app.mount(path=prefix, app=getattr(module, attr_name)) + + return _register + + +@dataclass(frozen=True) +class LazyFeature: + name: str + module_path: str + path_prefixes: Tuple[str, ...] + register_fn: Callable[["FastAPI", object], None] = field( + default_factory=lambda: _include_router("router") + ) + # For routes whose path has a leading parameter (e.g. /{server}/authorize) + # — startswith can't match those, so the matcher also checks endswith. + path_suffixes: Tuple[str, ...] = () + # Keep the stub injected even after load — for mounted ASGI sub-apps + # whose routes don't appear in the parent app's openapi spec. + persistent_swagger_stub: bool = False + + +LAZY_FEATURES: Tuple[LazyFeature, ...] = ( + LazyFeature( + name="guardrails", + module_path="litellm.proxy.guardrails.guardrail_endpoints", + path_prefixes=( + "/guardrails", + "/v2/guardrails", + "/apply_guardrail", + "/policies/usage", + ), + ), + LazyFeature( + name="policies", + module_path="litellm.proxy.management_endpoints.policy_endpoints", + # Trailing slash to avoid matching /policies/... (policy_engine). + path_prefixes=("/policy/", "/utils/test_policies_and_guardrails"), + ), + LazyFeature( + name="policy_engine", + module_path="litellm.proxy.policy_engine.policy_endpoints", + path_prefixes=("/policies",), + ), + LazyFeature( + name="policy_resolve", + module_path="litellm.proxy.policy_engine.policy_resolve_endpoints", + path_prefixes=("/policies/resolve", "/policies/attachments/estimate-impact"), + ), + LazyFeature( + name="agents", + module_path="litellm.proxy.agent_endpoints.endpoints", + path_prefixes=("/v1/agents", "/agents", "/agent/"), + ), + LazyFeature( + name="a2a", + module_path="litellm.proxy.agent_endpoints.a2a_endpoints", + path_prefixes=("/a2a", "/v1/a2a"), + ), + LazyFeature( + name="vector_stores", + module_path="litellm.proxy.vector_store_endpoints.endpoints", + path_prefixes=("/v1/vector_stores", "/vector_stores", "/v1/indexes"), + ), + LazyFeature( + name="vector_store_management", + module_path="litellm.proxy.vector_store_endpoints.management_endpoints", + # Trailing slash to avoid matching /vector_stores/... (vector_stores). + path_prefixes=("/vector_store/", "/v1/vector_store/"), + ), + LazyFeature( + name="vector_store_files", + # Routes appear under both /v1/vector_stores/{id}/files and the + # un-versioned form, so both prefixes must trigger the load. + module_path="litellm.proxy.vector_store_files_endpoints.endpoints", + path_prefixes=("/v1/vector_stores", "/vector_stores"), + ), + LazyFeature( + name="tools", + module_path="litellm.proxy.management_endpoints.tool_management_endpoints", + path_prefixes=("/v1/tool", "/tool"), + ), + LazyFeature( + name="search_tools", + module_path="litellm.proxy.search_endpoints.search_tool_management", + path_prefixes=("/search_tools",), + ), + # mcp_management owns most /v1/mcp/* admin routes; mcp_app is the mounted + # streaming sub-app at /mcp. + LazyFeature( + name="mcp_management", + module_path="litellm.proxy.management_endpoints.mcp_management_endpoints", + path_prefixes=("/v1/mcp/",), + ), + LazyFeature( + # Also serves /.well-known/oauth-* (OAuth metadata discovery). + # No /mcp/oauth prefix here: the mounted /mcp sub-app would + # shadow it, and there are no actual routes there anyway. + name="mcp_byok_oauth", + module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints", + path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"), + ), + LazyFeature( + # Serves OAuth dance endpoints (/authorize, /token, /callback, + # /register) plus several /.well-known/ discovery URLs at the proxy + # root — needed for MCP-over-OAuth flows even before /mcp is hit. + name="mcp_discoverable", + module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints", + path_prefixes=( + "/.well-known/oauth-", + "/.well-known/openid-configuration", + "/.well-known/jwks.json", + "/authorize", + "/token", + "/callback", + "/register", + ), + # Catches the /{mcp_server_name}/authorize|token|register variants. + path_suffixes=("/authorize", "/token", "/register"), + ), + LazyFeature( + name="mcp_rest", + module_path="litellm.proxy._experimental.mcp_server.rest_endpoints", + path_prefixes=("/mcp-rest",), + ), + LazyFeature( + # Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant + # here would defeat lazy loading. + name="mcp_app", + module_path="litellm.proxy._experimental.mcp_server.server", + path_prefixes=("/mcp",), + register_fn=_mount_app("/mcp", attr_name="app"), + persistent_swagger_stub=True, + ), + LazyFeature( + name="config_overrides", + module_path="litellm.proxy.management_endpoints.config_override_endpoints", + path_prefixes=("/config_overrides",), + ), + LazyFeature( + name="realtime", + module_path="litellm.proxy.realtime_endpoints.endpoints", + path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), + ), + LazyFeature( + name="anthropic_passthrough", + module_path="litellm.proxy.anthropic_endpoints.endpoints", + path_prefixes=("/v1/messages", "/anthropic", "/api/event_logging"), + ), + LazyFeature( + name="anthropic_skills", + module_path="litellm.proxy.anthropic_endpoints.skills_endpoints", + path_prefixes=("/v1/skills", "/skills"), + ), + LazyFeature( + name="langfuse_passthrough", + module_path="litellm.proxy.vertex_ai_endpoints.langfuse_endpoints", + path_prefixes=("/langfuse",), + ), + LazyFeature( + name="evals", + module_path="litellm.proxy.openai_evals_endpoints.endpoints", + path_prefixes=("/v1/evals", "/evals"), + ), + LazyFeature( + name="claude_code_marketplace", + module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints", + path_prefixes=("/claude-code",), + register_fn=_include_router("claude_code_marketplace_router"), + ), + LazyFeature( + name="scim", + module_path="litellm.proxy.management_endpoints.scim.scim_v2", + path_prefixes=("/scim",), + register_fn=_include_router("scim_router"), + ), + LazyFeature( + name="cloudzero", + module_path="litellm.proxy.spend_tracking.cloudzero_endpoints", + path_prefixes=("/cloudzero",), + ), + LazyFeature( + name="vantage", + module_path="litellm.proxy.spend_tracking.vantage_endpoints", + path_prefixes=("/vantage",), + ), + LazyFeature( + name="usage_ai", + module_path="litellm.proxy.management_endpoints.usage_endpoints", + path_prefixes=("/usage/ai",), + ), + LazyFeature( + name="prompts", + module_path="litellm.proxy.prompts.prompt_endpoints", + path_prefixes=("/prompts", "/utils/dotprompt_json_converter"), + ), + LazyFeature( + name="jwt_mappings", + module_path="litellm.proxy.management_endpoints.jwt_key_mapping_endpoints", + path_prefixes=("/jwt/key/mapping",), + ), + LazyFeature( + name="compliance", + module_path="litellm.proxy.management_endpoints.compliance_endpoints", + path_prefixes=("/compliance",), + ), + LazyFeature( + name="access_groups", + module_path="litellm.proxy.management_endpoints.access_group_endpoints", + path_prefixes=("/access_group", "/v1/access_group", "/v1/unified_access_group"), + ), +) + + +class LazyFeatureMiddleware: + """ASGI middleware that imports + registers a feature router on first + matching request. Idempotent; once loaded, subsequent requests skip.""" + + def __init__( + self, + app, + fastapi_app: "FastAPI", + features: Tuple[LazyFeature, ...] = LAZY_FEATURES, + ): + self.app = app + self._fastapi_app = fastapi_app + self._features = features + # Loaded set / per-feature locks live on app.state so the warm endpoint + # and the middleware share them — preventing duplicate registrations + # when both paths fire for the same feature. + if not hasattr(fastapi_app.state, "lazy_loaded"): + fastapi_app.state.lazy_loaded = set() + fastapi_app.state.lazy_locks = {} + + @property + def _loaded(self) -> set: + return self._fastapi_app.state.lazy_loaded + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + # Short-circuit once every feature has loaded. + if scope["type"] in ("http", "websocket") and len(self._loaded) < len( + self._features + ): + path = scope.get("path", "") + for feat in self._features: + if feat.module_path in self._loaded: + continue + if any(path.startswith(p) for p in feat.path_prefixes) or any( + path.endswith(s) for s in feat.path_suffixes + ): + await _force_load(self._fastapi_app, feat) + await self.app(scope, receive, send) + + +async def _force_load(app: "FastAPI", feat: LazyFeature) -> bool: + """Import + register a lazy feature exactly once per (app, module). + Shared by the middleware and the /lazy/warm endpoint.""" + if not hasattr(app.state, "lazy_loaded"): + app.state.lazy_loaded = set() + app.state.lazy_locks = {} + lock = app.state.lazy_locks.setdefault(feat.module_path, asyncio.Lock()) + async with lock: + if feat.module_path in app.state.lazy_loaded: + return False + try: + # Import on a thread (heavy modules take 1-3 s). register_fn + # mutates app.router.routes, so it stays on the loop thread. + loop = asyncio.get_running_loop() + module = await loop.run_in_executor( + None, importlib.import_module, feat.module_path + ) + feat.register_fn(app, module) + app.state.lazy_loaded.add(feat.module_path) + app.openapi_schema = None + verbose_proxy_logger.info( + "Lazy-loaded optional feature %r (module: %s)", + feat.name, + feat.module_path, + ) + return True + except Exception as exc: + # Mark loaded anyway so we don't retry on every request. + app.state.lazy_loaded.add(feat.module_path) + verbose_proxy_logger.warning( + "Failed to lazy-load optional feature %r (module: %s): %s. " + "This feature's endpoints will return 404 until restart.", + feat.name, + feat.module_path, + exc, + ) + return False + + +def attach_lazy_features(app: "FastAPI") -> None: + app.include_router(_make_warmup_router(app)) + app.add_middleware(LazyFeatureMiddleware, fastapi_app=app) + + +def _make_warmup_router(app: "FastAPI") -> "APIRouter": + """POST /lazy/warm/{name}: load a feature and return its partial openapi + so the Swagger plugin can merge in-place without a full /openapi.json refetch. + Requires auth — anyone who can hit the proxy can already trigger the same + imports by sending a real request to a feature's prefix, but gating this + debug endpoint avoids unauthenticated callers forcing the import chain.""" + from fastapi import APIRouter, Depends, HTTPException + from fastapi.openapi.utils import get_openapi + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + router = APIRouter() + + @router.post( + "/lazy/warm/{name}", + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], + ) + async def warm(name: str): + feat = next((f for f in LAZY_FEATURES if f.name == name), None) + if feat is None: + raise HTTPException(404, f"unknown lazy feature: {name}") + if feat.persistent_swagger_stub: + return {"stub_path": None, "paths": {}, "components": {"schemas": {}}} + + await _force_load(app, feat) + + feat_routes = [ + r + for r in app.routes + if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes) + ] + full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + # Force all operations under one tag so they group under a single Swagger + # section — many lazy modules tag routes inconsistently. + for path_ops in full.get("paths", {}).values(): + for op in path_ops.values(): + if isinstance(op, dict): + op["tags"] = [feat.name] + return { + "stub_path": feat.path_prefixes[0], + "paths": full.get("paths", {}), + "components": {"schemas": full.get("components", {}).get("schemas", {})}, + } + + return router + + +def inject_lazy_stubs(schema: Dict) -> Dict: + """Inject openapi entries for unloaded features. Uses the snapshot file + when available (full route info), otherwise falls back to a single + placeholder per feature. Any failure logs and returns the schema unchanged + so /openapi.json never 500s on a cosmetic injection bug.""" + try: + from litellm.proxy._lazy_openapi_snapshot import load_snapshot + + snapshot = load_snapshot() + paths = schema.setdefault("paths", {}) + schemas = schema.setdefault("components", {}).setdefault("schemas", {}) + + for feat in LAZY_FEATURES: + if feat.module_path in sys.modules and not feat.persistent_swagger_stub: + continue + + fragment = (snapshot or {}).get(feat.name) + if fragment: + for p, ops in fragment.get("paths", {}).items(): + paths.setdefault(p, ops) + for name, sch in ( + fragment.get("components", {}).get("schemas", {}).items() + ): + schemas.setdefault(name, sch) + continue + + prefix = feat.path_prefixes[0] + if prefix in paths: + continue + paths[prefix] = { + "get": { + "tags": [feat.name], + "summary": feat.name, + "responses": {"200": {"description": "OK"}}, + } + } + except Exception as exc: + verbose_proxy_logger.warning("inject_lazy_stubs failed: %s", exc) + return schema + + +def lazy_tag_to_prefix() -> Dict[str, str]: + """feature.name -> first prefix, used by the Swagger warmup JS plugin. + Returns empty when the snapshot is loaded — the plugin is unnecessary + because /openapi.json already has full route info.""" + from litellm.proxy._lazy_openapi_snapshot import load_snapshot + + if load_snapshot(): + return {} + return { + feat.name: feat.path_prefixes[0] + for feat in LAZY_FEATURES + if not feat.persistent_swagger_stub + } diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json new file mode 100644 index 0000000000..8331f748c6 --- /dev/null +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -0,0 +1,31651 @@ +{ + "a2a": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/a2a/{agent_id}": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_a2a__agent_id__post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/.well-known/agent-card.json": { + "get": { + "description": "Get the agent card for an agent (A2A discovery endpoint).\n\nSupports both standard paths:\n- /.well-known/agent-card.json\n- /.well-known/agent.json\n\nThe URL in the agent card is rewritten to point to the LiteLLM proxy,\nso all subsequent A2A calls go through LiteLLM for logging and cost tracking.", + "operationId": "get_agent_card_a2a__agent_id___well_known_agent_card_json_get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Card", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/.well-known/agent.json": { + "get": { + "description": "Get the agent card for an agent (A2A discovery endpoint).\n\nSupports both standard paths:\n- /.well-known/agent-card.json\n- /.well-known/agent.json\n\nThe URL in the agent card is rewritten to point to the LiteLLM proxy,\nso all subsequent A2A calls go through LiteLLM for logging and cost tracking.", + "operationId": "get_agent_card_a2a__agent_id___well_known_agent_json_get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Card", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/message/send": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_a2a__agent_id__message_send_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + }, + "/v1/a2a/{agent_id}/message/send": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_v1_a2a__agent_id__message_send_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + } + } + }, + "access_groups": { + "components": { + "schemas": { + "AccessGroupCreateRequest": { + "properties": { + "access_agent_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Agent Ids" + }, + "access_group_name": { + "title": "Access Group Name", + "type": "string" + }, + "access_mcp_server_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Mcp Server Ids" + }, + "access_model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Model Names" + }, + "assigned_key_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Key Ids" + }, + "assigned_team_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Team Ids" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + } + }, + "required": [ + "access_group_name" + ], + "title": "AccessGroupCreateRequest", + "type": "object" + }, + "AccessGroupInfo": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "deployment_count": { + "title": "Deployment Count", + "type": "integer" + }, + "model_names": { + "items": { + "type": "string" + }, + "title": "Model Names", + "type": "array" + } + }, + "required": [ + "access_group", + "model_names", + "deployment_count" + ], + "title": "AccessGroupInfo", + "type": "object" + }, + "AccessGroupResponse": { + "properties": { + "access_agent_ids": { + "items": { + "type": "string" + }, + "title": "Access Agent Ids", + "type": "array" + }, + "access_group_id": { + "title": "Access Group Id", + "type": "string" + }, + "access_group_name": { + "title": "Access Group Name", + "type": "string" + }, + "access_mcp_server_ids": { + "items": { + "type": "string" + }, + "title": "Access Mcp Server Ids", + "type": "array" + }, + "access_model_names": { + "items": { + "type": "string" + }, + "title": "Access Model Names", + "type": "array" + }, + "assigned_key_ids": { + "items": { + "type": "string" + }, + "title": "Assigned Key Ids", + "type": "array" + }, + "assigned_team_ids": { + "items": { + "type": "string" + }, + "title": "Assigned Team Ids", + "type": "array" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "access_group_id", + "access_group_name", + "access_model_names", + "access_mcp_server_ids", + "access_agent_ids", + "assigned_team_ids", + "assigned_key_ids", + "created_at", + "updated_at" + ], + "title": "AccessGroupResponse", + "type": "object" + }, + "AccessGroupUpdateRequest": { + "properties": { + "access_agent_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Agent Ids" + }, + "access_group_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Access Group Name" + }, + "access_mcp_server_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Mcp Server Ids" + }, + "access_model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Model Names" + }, + "assigned_key_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Key Ids" + }, + "assigned_team_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Team Ids" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + } + }, + "title": "AccessGroupUpdateRequest", + "type": "object" + }, + "DeleteModelGroupResponse": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "models_updated": { + "title": "Models Updated", + "type": "integer" + } + }, + "required": [ + "access_group", + "models_updated", + "message" + ], + "title": "DeleteModelGroupResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListAccessGroupsResponse": { + "properties": { + "access_groups": { + "items": { + "$ref": "#/components/schemas/AccessGroupInfo" + }, + "title": "Access Groups", + "type": "array" + } + }, + "required": [ + "access_groups" + ], + "title": "ListAccessGroupsResponse", + "type": "object" + }, + "NewModelGroupRequest": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + } + }, + "required": [ + "access_group" + ], + "title": "NewModelGroupRequest", + "type": "object" + }, + "NewModelGroupResponse": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + }, + "models_updated": { + "title": "Models Updated", + "type": "integer" + } + }, + "required": [ + "access_group", + "models_updated" + ], + "title": "NewModelGroupResponse", + "type": "object" + }, + "UpdateModelGroupRequest": { + "properties": { + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + } + }, + "title": "UpdateModelGroupRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/access_group/list": { + "get": { + "description": "List all access groups.\n\nReturns a list of all access groups with their model names and deployment counts.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/list' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nReturns:\n- ListAccessGroupsResponse with all access groups", + "operationId": "list_access_groups_access_group_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListAccessGroupsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/new": { + "post": { + "description": "Create a new access group containing multiple model names.\n\nAn access group is a named collection of model groups that can be referenced\nby teams/keys for simplified access control.\n\nExample:\n```bash\ncurl -X POST 'http://localhost:4000/access_group/new' \\\n -H 'Authorization: Bearer sk-1234' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"access_group\": \"production-models\",\n \"model_names\": [\"gpt-4\", \"claude-3-opus\", \"gemini-pro\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (e.g., \"production-models\")\n- model_names: List[str] - List of existing model groups to include\n\nReturns:\n- NewModelGroupResponse with the created access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 500: If database operations fail", + "operationId": "create_model_group_access_group_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Model Group", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/delete": { + "delete": { + "description": "Delete an access group.\n\nRemoves the access group from all deployments that have it.\n\nExample:\n```bash\ncurl -X DELETE 'http://localhost:4000/access_group/production-models/delete' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- DeleteModelGroupResponse with deletion details\n\nRaises:\n- HTTPException 404: If access group not found", + "operationId": "delete_access_group_access_group__access_group__delete_delete", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/info": { + "get": { + "description": "Get information about a specific access group.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/production-models/info' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- AccessGroupInfo with the access group details\n\nRaises:\n- HTTPException 404: If access group not found", + "operationId": "get_access_group_info_access_group__access_group__info_get", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupInfo" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group Info", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/update": { + "put": { + "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H 'Authorization: Bearer sk-1234' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found", + "operationId": "update_access_group_access_group__access_group__update_put", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateModelGroupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/access_group": { + "get": { + "operationId": "list_access_groups_v1_access_group_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AccessGroupResponse" + }, + "title": "Response List Access Groups V1 Access Group Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + }, + "post": { + "operationId": "create_access_group_v1_access_group_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/access_group/{access_group_id}": { + "delete": { + "operationId": "delete_access_group_v1_access_group__access_group_id__delete", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + }, + "get": { + "operationId": "get_access_group_v1_access_group__access_group_id__get", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group", + "tags": [ + "access_groups" + ] + }, + "put": { + "operationId": "update_access_group_v1_access_group__access_group_id__put", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/unified_access_group": { + "get": { + "operationId": "list_access_groups_v1_unified_access_group_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AccessGroupResponse" + }, + "title": "Response List Access Groups V1 Unified Access Group Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + }, + "post": { + "operationId": "create_access_group_v1_unified_access_group_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/unified_access_group/{access_group_id}": { + "delete": { + "operationId": "delete_access_group_v1_unified_access_group__access_group_id__delete", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + }, + "get": { + "operationId": "get_access_group_v1_unified_access_group__access_group_id__get", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group", + "tags": [ + "access_groups" + ] + }, + "put": { + "operationId": "update_access_group_v1_unified_access_group__access_group_id__put", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + } + } + }, + "agents": { + "components": { + "schemas": { + "APIKeySecurityScheme": { + "description": "Defines a security scheme using an API key.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "in_": { + "enum": [ + "query", + "header", + "cookie" + ], + "title": "In", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "type": { + "const": "apiKey", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "in_", + "name" + ], + "title": "APIKeySecurityScheme", + "type": "object" + }, + "AgentCapabilities": { + "description": "Defines optional capabilities supported by an agent.", + "properties": { + "extensions": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentExtension" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extensions" + }, + "pushNotifications": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Pushnotifications" + }, + "stateTransitionHistory": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Statetransitionhistory" + }, + "streaming": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Streaming" + } + }, + "title": "AgentCapabilities", + "type": "object" + }, + "AgentCard": { + "description": "The AgentCard is a self-describing manifest for an agent.\nIt provides essential metadata including the agent's identity, capabilities,\nskills, supported communication methods, and security requirements.", + "properties": { + "additionalInterfaces": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentInterface" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Additionalinterfaces" + }, + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities" + }, + "defaultInputModes": { + "items": { + "type": "string" + }, + "title": "Defaultinputmodes", + "type": "array" + }, + "defaultOutputModes": { + "items": { + "type": "string" + }, + "title": "Defaultoutputmodes", + "type": "array" + }, + "description": { + "title": "Description", + "type": "string" + }, + "documentationUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Documentationurl" + }, + "iconUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Iconurl" + }, + "name": { + "title": "Name", + "type": "string" + }, + "preferredTransport": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Preferredtransport" + }, + "protocolVersion": { + "title": "Protocolversion", + "type": "string" + }, + "provider": { + "anyOf": [ + { + "$ref": "#/components/schemas/AgentProvider" + }, + { + "type": "null" + } + ] + }, + "security": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Security" + }, + "securitySchemes": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/components/schemas/APIKeySecurityScheme" + }, + { + "$ref": "#/components/schemas/HTTPAuthSecurityScheme" + }, + { + "$ref": "#/components/schemas/OAuth2SecurityScheme" + }, + { + "$ref": "#/components/schemas/OpenIdConnectSecurityScheme" + }, + { + "$ref": "#/components/schemas/MutualTLSSecurityScheme" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Securityschemes" + }, + "signatures": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentCardSignature" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Signatures" + }, + "skills": { + "items": { + "$ref": "#/components/schemas/AgentSkill" + }, + "title": "Skills", + "type": "array" + }, + "supportsAuthenticatedExtendedCard": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Supportsauthenticatedextendedcard" + }, + "url": { + "title": "Url", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "title": "AgentCard", + "type": "object" + }, + "AgentCardSignature": { + "description": "Represents a JWS signature of an AgentCard.", + "properties": { + "header": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Header" + }, + "protected": { + "title": "Protected", + "type": "string" + }, + "signature": { + "title": "Signature", + "type": "string" + } + }, + "title": "AgentCardSignature", + "type": "object" + }, + "AgentConfig": { + "properties": { + "agent_card_params": { + "$ref": "#/components/schemas/AgentCard" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "object_permission": { + "$ref": "#/components/schemas/AgentObjectPermission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + } + }, + "required": [ + "agent_name", + "agent_card_params" + ], + "title": "AgentConfig", + "type": "object" + }, + "AgentExtension": { + "description": "A declaration of a protocol extension supported by an Agent.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Params" + }, + "required": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Required" + }, + "uri": { + "title": "Uri", + "type": "string" + } + }, + "title": "AgentExtension", + "type": "object" + }, + "AgentInterface": { + "description": "Declares a combination of a target URL and a transport protocol.", + "properties": { + "transport": { + "title": "Transport", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "title": "AgentInterface", + "type": "object" + }, + "AgentMakePublicResponse": { + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "public_agent_groups": { + "items": { + "type": "string" + }, + "title": "Public Agent Groups", + "type": "array" + }, + "updated_by": { + "title": "Updated By", + "type": "string" + } + }, + "required": [ + "message", + "public_agent_groups", + "updated_by" + ], + "title": "AgentMakePublicResponse", + "type": "object" + }, + "AgentObjectPermission": { + "properties": { + "agents": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Agents" + }, + "mcp_access_groups": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Mcp Access Groups" + }, + "mcp_servers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Mcp Servers" + }, + "mcp_tool_permissions": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Tool Permissions" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Models" + } + }, + "title": "AgentObjectPermission", + "type": "object" + }, + "AgentProvider": { + "description": "Represents the service provider of an agent.", + "properties": { + "organization": { + "title": "Organization", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "title": "AgentProvider", + "type": "object" + }, + "AgentResponse": { + "properties": { + "agent_card_params": { + "additionalProperties": true, + "title": "Agent Card Params", + "type": "object" + }, + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "object_permission": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Object Permission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "spend": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Spend" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "agent_id", + "agent_name", + "agent_card_params" + ], + "title": "AgentResponse", + "type": "object" + }, + "AgentSkill": { + "description": "Represents a distinct capability or function that an agent can perform.", + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "examples": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Examples" + }, + "id": { + "title": "Id", + "type": "string" + }, + "inputModes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Inputmodes" + }, + "name": { + "title": "Name", + "type": "string" + }, + "outputModes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Outputmodes" + }, + "security": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Security" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "title": "AgentSkill", + "type": "object" + }, + "BreakdownMetrics": { + "description": "Breakdown of spend by different dimensions", + "properties": { + "api_keys": { + "additionalProperties": { + "$ref": "#/components/schemas/KeyMetricWithMetadata" + }, + "title": "Api Keys", + "type": "object" + }, + "endpoints": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Endpoints", + "type": "object" + }, + "entities": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Entities", + "type": "object" + }, + "mcp_servers": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Mcp Servers", + "type": "object" + }, + "model_groups": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Model Groups", + "type": "object" + }, + "models": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Models", + "type": "object" + }, + "providers": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Providers", + "type": "object" + } + }, + "title": "BreakdownMetrics", + "type": "object" + }, + "DailySpendData": { + "properties": { + "breakdown": { + "$ref": "#/components/schemas/BreakdownMetrics" + }, + "date": { + "format": "date", + "title": "Date", + "type": "string" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "date", + "metrics" + ], + "title": "DailySpendData", + "type": "object" + }, + "DailySpendMetadata": { + "properties": { + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "page": { + "default": 1, + "title": "Page", + "type": "integer" + }, + "total_api_requests": { + "default": 0, + "title": "Total Api Requests", + "type": "integer" + }, + "total_cache_creation_input_tokens": { + "default": 0, + "title": "Total Cache Creation Input Tokens", + "type": "integer" + }, + "total_cache_read_input_tokens": { + "default": 0, + "title": "Total Cache Read Input Tokens", + "type": "integer" + }, + "total_completion_tokens": { + "default": 0, + "title": "Total Completion Tokens", + "type": "integer" + }, + "total_failed_requests": { + "default": 0, + "title": "Total Failed Requests", + "type": "integer" + }, + "total_pages": { + "default": 1, + "title": "Total Pages", + "type": "integer" + }, + "total_prompt_tokens": { + "default": 0, + "title": "Total Prompt Tokens", + "type": "integer" + }, + "total_spend": { + "default": 0.0, + "title": "Total Spend", + "type": "number" + }, + "total_successful_requests": { + "default": 0, + "title": "Total Successful Requests", + "type": "integer" + }, + "total_tokens": { + "default": 0, + "title": "Total Tokens", + "type": "integer" + } + }, + "title": "DailySpendMetadata", + "type": "object" + }, + "HTTPAuthSecurityScheme": { + "description": "Defines a security scheme using HTTP authentication.", + "properties": { + "bearerFormat": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Bearerformat" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "scheme": { + "title": "Scheme", + "type": "string" + }, + "type": { + "const": "http", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "scheme", + "bearerFormat" + ], + "title": "HTTPAuthSecurityScheme", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "KeyMetadata": { + "description": "Metadata for a key", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + "title": "KeyMetadata", + "type": "object" + }, + "KeyMetricWithMetadata": { + "description": "Base class for metrics with additional metadata", + "properties": { + "metadata": { + "$ref": "#/components/schemas/KeyMetadata" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "metrics" + ], + "title": "KeyMetricWithMetadata", + "type": "object" + }, + "MakeAgentsPublicRequest": { + "properties": { + "agent_ids": { + "items": { + "type": "string" + }, + "title": "Agent Ids", + "type": "array" + } + }, + "required": [ + "agent_ids" + ], + "title": "MakeAgentsPublicRequest", + "type": "object" + }, + "MetricWithMetadata": { + "properties": { + "api_key_breakdown": { + "additionalProperties": { + "$ref": "#/components/schemas/KeyMetricWithMetadata" + }, + "title": "Api Key Breakdown", + "type": "object" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "metrics" + ], + "title": "MetricWithMetadata", + "type": "object" + }, + "MutualTLSSecurityScheme": { + "description": "Defines a security scheme using mTLS authentication.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "type": { + "const": "mutualTLS", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "MutualTLSSecurityScheme", + "type": "object" + }, + "OAuth2SecurityScheme": { + "description": "Defines a security scheme using OAuth 2.0.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "flows": { + "$ref": "#/components/schemas/OAuthFlows" + }, + "oauth2MetadataUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2Metadataurl" + }, + "type": { + "const": "oauth2", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "flows", + "oauth2MetadataUrl" + ], + "title": "OAuth2SecurityScheme", + "type": "object" + }, + "OAuthFlows": { + "description": "Defines the configuration for the supported OAuth 2.0 flows.", + "properties": { + "authorizationCode": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Authorizationcode" + }, + "clientCredentials": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Clientcredentials" + }, + "implicit": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Implicit" + }, + "password": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Password" + } + }, + "title": "OAuthFlows", + "type": "object" + }, + "OpenIdConnectSecurityScheme": { + "description": "Defines a security scheme using OpenID Connect.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "openIdConnectUrl": { + "title": "Openidconnecturl", + "type": "string" + }, + "type": { + "const": "openIdConnect", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "openIdConnectUrl" + ], + "title": "OpenIdConnectSecurityScheme", + "type": "object" + }, + "PatchAgentRequest": { + "properties": { + "agent_card_params": { + "$ref": "#/components/schemas/AgentCard" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "object_permission": { + "$ref": "#/components/schemas/AgentObjectPermission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + } + }, + "title": "PatchAgentRequest", + "type": "object" + }, + "SpendAnalyticsPaginatedResponse": { + "properties": { + "metadata": { + "$ref": "#/components/schemas/DailySpendMetadata" + }, + "results": { + "items": { + "$ref": "#/components/schemas/DailySpendData" + }, + "title": "Results", + "type": "array" + } + }, + "required": [ + "results" + ], + "title": "SpendAnalyticsPaginatedResponse", + "type": "object" + }, + "SpendMetrics": { + "properties": { + "api_requests": { + "default": 0, + "title": "Api Requests", + "type": "integer" + }, + "cache_creation_input_tokens": { + "default": 0, + "title": "Cache Creation Input Tokens", + "type": "integer" + }, + "cache_read_input_tokens": { + "default": 0, + "title": "Cache Read Input Tokens", + "type": "integer" + }, + "completion_tokens": { + "default": 0, + "title": "Completion Tokens", + "type": "integer" + }, + "failed_requests": { + "default": 0, + "title": "Failed Requests", + "type": "integer" + }, + "prompt_tokens": { + "default": 0, + "title": "Prompt Tokens", + "type": "integer" + }, + "spend": { + "default": 0.0, + "title": "Spend", + "type": "number" + }, + "successful_requests": { + "default": 0, + "title": "Successful Requests", + "type": "integer" + }, + "total_tokens": { + "default": 0, + "title": "Total Tokens", + "type": "integer" + } + }, + "title": "SpendMetrics", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/agent/daily/activity": { + "get": { + "description": "Get daily activity for specific agents or all accessible agents.", + "operationId": "get_agent_daily_activity_agent_daily_activity_get", + "parameters": [ + { + "in": "query", + "name": "agent_ids", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Agent Ids" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + }, + { + "in": "query", + "name": "model", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + } + }, + { + "in": "query", + "name": "api_key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 10, + "title": "Page Size", + "type": "integer" + } + }, + { + "in": "query", + "name": "exclude_agent_ids", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Exclude Agent Ids" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SpendAnalyticsPaginatedResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Daily Activity", + "tags": [ + "agents" + ] + } + }, + "/v1/agents": { + "get": { + "description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]", + "operationId": "get_agents_v1_agents_get", + "parameters": [ + { + "description": "When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.", + "in": "query", + "name": "health_check", + "required": false, + "schema": { + "default": false, + "description": "When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.", + "title": "Health Check", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AgentResponse" + }, + "title": "Response Get Agents V1 Agents Get", + "type": "array" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agents", + "tags": [ + "agents" + ] + }, + "post": { + "description": "Create a new agent\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/agents\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"my-custom-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Hello World Agent\",\n \"description\": \"Just a hello world agent\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.0.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": [\n {\n \"id\": \"hello_world\",\n \"name\": \"Returns hello world\",\n \"description\": \"just returns hello world\",\n \"tags\": [\"hello world\"],\n \"examples\": [\"hi\", \"hello world\"]\n }\n ]\n },\n \"litellm_params\": {\n \"make_public\": true\n }\n }\n }'\n```", + "operationId": "create_agent_v1_agents_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Agent", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/make_public": { + "post": { + "description": "Make multiple agents publicly discoverable\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/v1/agents/make_public\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent_ids\": [\"123e4567-e89b-12d3-a456-426614174000\", \"123e4567-e89b-12d3-a456-426614174001\"]\n }'\n```\n\nExample Response:\n```json\n{\n \"agent_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"agent_name\": \"my-custom-agent\",\n \"litellm_params\": {\n \"make_public\": true\n },\n \"agent_card_params\": {...},\n \"created_at\": \"2025-11-15T10:30:00Z\",\n \"updated_at\": \"2025-11-15T10:35:00Z\",\n \"created_by\": \"user123\",\n \"updated_by\": \"user123\"\n}\n```", + "operationId": "make_agents_public_v1_agents_make_public_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MakeAgentsPublicRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMakePublicResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Agents Public", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/{agent_id}": { + "delete": { + "description": "Delete an agent\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Agent 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_agent_v1_agents__agent_id__delete", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Agent", + "tags": [ + "agents" + ] + }, + "get": { + "description": "Get a specific agent by ID\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_agent_by_id_v1_agents__agent_id__get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent By Id", + "tags": [ + "agents" + ] + }, + "patch": { + "description": "Update an existing agent\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"updated-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Updated Agent\",\n \"description\": \"Updated description\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.1.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": []\n },\n \"litellm_params\": {\n \"make_public\": false\n }\n }\n }'\n```", + "operationId": "patch_agent_v1_agents__agent_id__patch", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchAgentRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Agent", + "tags": [ + "agents" + ] + }, + "put": { + "description": "Update an existing agent\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"updated-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Updated Agent\",\n \"description\": \"Updated description\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.1.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": []\n },\n \"litellm_params\": {\n \"make_public\": false\n }\n }\n }'\n```", + "operationId": "update_agent_v1_agents__agent_id__put", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Agent", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/{agent_id}/make_public": { + "post": { + "description": "Make an agent publicly discoverable\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000/make_public\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\"\n```\n\nExample Response:\n```json\n{\n \"agent_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"agent_name\": \"my-custom-agent\",\n \"litellm_params\": {\n \"make_public\": true\n },\n \"agent_card_params\": {...},\n \"created_at\": \"2025-11-15T10:30:00Z\",\n \"updated_at\": \"2025-11-15T10:35:00Z\",\n \"created_by\": \"user123\",\n \"updated_by\": \"user123\"\n}\n```", + "operationId": "make_agent_public_v1_agents__agent_id__make_public_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMakePublicResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Agent Public", + "tags": [ + "agents" + ] + } + } + } + }, + "anthropic_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/anthropic/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/api/event_logging/batch": { + "post": { + "description": "Stubbed endpoint for Anthropic event logging batch requests.\n\nThis endpoint accepts event logging requests but does nothing with them.\nIt exists to prevent 404 errors from Claude Code clients that send telemetry.", + "operationId": "event_logging_batch_api_event_logging_batch_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Event Logging Batch", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/v1/messages": { + "post": { + "description": "Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion).\n\nThis was a BETA endpoint that calls 100+ LLMs in the anthropic format.", + "operationId": "anthropic_response_v1_messages_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Response", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/v1/messages/count_tokens": { + "post": { + "description": "Count tokens for Anthropic Messages API format.\n\nThis endpoint follows the Anthropic Messages API token counting specification.\nIt accepts the same parameters as the /v1/messages endpoint but returns\ntoken counts instead of generating a response.\n\nExample usage:\n```\ncurl -X POST \"http://localhost:4000/v1/messages/count_tokens?beta=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" -d '{\n \"model\": \"claude-3-sonnet-20240229\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello Claude!\"}]\n }'\n```\n\nReturns: {\"input_tokens\": }", + "operationId": "count_tokens_v1_messages_count_tokens_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Count Tokens", + "tags": [ + "anthropic_passthrough" + ] + } + } + } + }, + "anthropic_skills": { + "components": { + "schemas": { + "DeleteSkillResponse": { + "description": "Response from deleting a skill", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "type": { + "default": "skill_deleted", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id" + ], + "title": "DeleteSkillResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListSkillsResponse": { + "description": "Response from listing skills", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Skill" + }, + "title": "Data", + "type": "array" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "next_page": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Page" + } + }, + "required": [ + "data" + ], + "title": "ListSkillsResponse", + "type": "object" + }, + "Skill": { + "description": "Represents a skill from the Anthropic Skills API", + "properties": { + "created_at": { + "title": "Created At", + "type": "string" + }, + "display_title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display Title" + }, + "id": { + "title": "Id", + "type": "string" + }, + "latest_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Latest Version" + }, + "source": { + "title": "Source", + "type": "string" + }, + "type": { + "default": "skill", + "title": "Type", + "type": "string" + }, + "updated_at": { + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "created_at", + "source", + "updated_at" + ], + "title": "Skill", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/skills": { + "get": { + "description": "List skills on Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl \"http://localhost:4000/v1/skills?beta=true&limit=10\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl \"http://localhost:4000/v1/skills?beta=true&limit=10\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: ListSkillsResponse with list of skills", + "operationId": "list_skills_v1_skills_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After Id" + } + }, + { + "in": "query", + "name": "before_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before Id" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSkillsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Skills", + "tags": [ + "anthropic_skills" + ] + }, + "post": { + "description": "Create a new skill on Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via form field: `model=claude-account-1`\n\nExample usage:\n```bash\n# Basic usage\ncurl -X POST \"http://localhost:4000/v1/skills?beta=true\" -H \"Content-Type: multipart/form-data\" -H \"Authorization: Bearer your-key\" -F \"display_title=My Skill\" -F \"files[]=@skill.zip\"\n\n# With model-based routing\ncurl -X POST \"http://localhost:4000/v1/skills?beta=true\" -H \"Content-Type: multipart/form-data\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\" -F \"display_title=My Skill\" -F \"files[]=@skill.zip\"\n```\n\nReturns: Skill object with id, display_title, etc.", + "operationId": "create_skill_v1_skills_post", + "parameters": [ + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Skill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Skill", + "tags": [ + "anthropic_skills" + ] + } + }, + "/v1/skills/{skill_id}": { + "delete": { + "description": "Delete a skill by ID from Anthropic.\n\nRequires `?beta=true` query parameter.\n\nNote: Anthropic does not allow deleting skills with existing versions.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl -X DELETE \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl -X DELETE \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: DeleteSkillResponse with type=\"skill_deleted\"", + "operationId": "delete_skill_v1_skills__skill_id__delete", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteSkillResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Skill", + "tags": [ + "anthropic_skills" + ] + }, + "get": { + "description": "Get a specific skill by ID from Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: Skill object", + "operationId": "get_skill_v1_skills__skill_id__get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Skill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Skill", + "tags": [ + "anthropic_skills" + ] + } + } + } + }, + "claude_code_marketplace": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListPluginsResponse": { + "description": "Response from listing plugins.", + "properties": { + "count": { + "title": "Count", + "type": "integer" + }, + "plugins": { + "items": { + "$ref": "#/components/schemas/PluginListItem" + }, + "title": "Plugins", + "type": "array" + } + }, + "required": [ + "plugins", + "count" + ], + "title": "ListPluginsResponse", + "type": "object" + }, + "PluginAuthor": { + "description": "Plugin author information.", + "properties": { + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Author email", + "title": "Email" + }, + "name": { + "description": "Author name", + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "PluginAuthor", + "type": "object" + }, + "PluginListItem": { + "description": "Plugin item in list responses.", + "properties": { + "author": { + "anyOf": [ + { + "$ref": "#/components/schemas/PluginAuthor" + }, + { + "type": "null" + } + ] + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "homepage": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Homepage" + }, + "id": { + "title": "Id", + "type": "string" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "name": { + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "title": "Source", + "type": "object" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "id", + "name", + "version", + "description", + "source", + "enabled", + "created_at", + "updated_at" + ], + "title": "PluginListItem", + "type": "object" + }, + "RegisterPluginRequest": { + "description": "Request body for registering a plugin in the marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket and referenced by their git source.", + "properties": { + "author": { + "anyOf": [ + { + "$ref": "#/components/schemas/PluginAuthor" + }, + { + "type": "null" + } + ], + "description": "Plugin author" + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin category", + "title": "Category" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin description", + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill domain (e.g., 'Productivity')", + "title": "Domain" + }, + "homepage": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin homepage URL", + "title": "Homepage" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Search keywords", + "title": "Keywords" + }, + "name": { + "description": "Plugin name (kebab-case, e.g., 'my-plugin')", + "pattern": "^[a-z0-9-]+$", + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill namespace within domain (e.g., 'workflows')", + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "description": "Git source reference. Supported formats:\n- GitHub: {'source': 'github', 'repo': 'org/repo'}\n- Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'}\n- Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'}", + "title": "Source", + "type": "object" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "1.0.0", + "description": "Semantic version", + "title": "Version" + } + }, + "required": [ + "name", + "source" + ], + "title": "RegisterPluginRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/claude-code/marketplace.json": { + "get": { + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin install my-plugin@litellm\n ```", + "operationId": "get_marketplace_claude_code_marketplace_json_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Marketplace", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins": { + "get": { + "description": "List all plugins in the marketplace.\n\nParameters:\n - enabled_only: If true, only return enabled plugins\n\nReturns:\n List of plugins with their metadata.", + "operationId": "list_plugins_claude_code_plugins_get", + "parameters": [ + { + "in": "query", + "name": "enabled_only", + "required": false, + "schema": { + "default": false, + "title": "Enabled Only", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPluginsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Plugins", + "tags": [ + "claude_code_marketplace" + ] + }, + "post": { + "description": "Register a plugin in the LiteLLM marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket. Claude Code will clone from the git source\nwhen users install.\n\nParameters:\n - name: Plugin name (kebab-case)\n - source: Git source reference (github, url, or git-subdir format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n\nReturns:\n Registration status and plugin information.\n\nExample:\n ```bash\n curl -X POST http://localhost:4000/claude-code/plugins \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-plugin\",\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"1.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", + "operationId": "register_plugin_claude_code_plugins_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}": { + "delete": { + "description": "Delete a plugin from the marketplace.\n\nParameters:\n - plugin_name: The name of the plugin to delete", + "operationId": "delete_plugin_claude_code_plugins__plugin_name__delete", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Plugin", + "tags": [ + "claude_code_marketplace" + ] + }, + "get": { + "description": "Get details of a specific plugin.\n\nParameters:\n - plugin_name: The name of the plugin\n\nReturns:\n Plugin details including source and metadata.", + "operationId": "get_plugin_claude_code_plugins__plugin_name__get", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}/disable": { + "post": { + "description": "Disable a plugin without deleting it.\n\nParameters:\n - plugin_name: The name of the plugin to disable", + "operationId": "disable_plugin_claude_code_plugins__plugin_name__disable_post", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Disable Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}/enable": { + "post": { + "description": "Enable a disabled plugin.\n\nParameters:\n - plugin_name: The name of the plugin to enable", + "operationId": "enable_plugin_claude_code_plugins__plugin_name__enable_post", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enable Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + } + } + }, + "cloudzero": { + "components": { + "schemas": { + "CloudZeroExportRequest": { + "description": "Request model for CloudZero export operations", + "properties": { + "end_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "End time for data export in UTC", + "title": "End Time Utc" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Optional limit on number of records to export", + "title": "Limit" + }, + "operation": { + "default": "replace_hourly", + "description": "CloudZero operation type (replace_hourly or sum)", + "title": "Operation", + "type": "string" + }, + "start_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Start time for data export in UTC", + "title": "Start Time Utc" + } + }, + "title": "CloudZeroExportRequest", + "type": "object" + }, + "CloudZeroExportResponse": { + "description": "Response model for CloudZero export operations", + "properties": { + "dry_run_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dry run data including usage data and CBF transformed data", + "title": "Dry Run Data" + }, + "message": { + "title": "Message", + "type": "string" + }, + "records_exported": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Records Exported" + }, + "status": { + "title": "Status", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Summary statistics for dry run", + "title": "Summary" + } + }, + "required": [ + "message", + "status" + ], + "title": "CloudZeroExportResponse", + "type": "object" + }, + "CloudZeroInitRequest": { + "description": "Request model for initializing CloudZero settings", + "properties": { + "api_key": { + "description": "CloudZero API key for authentication", + "title": "Api Key", + "type": "string" + }, + "connection_id": { + "description": "CloudZero connection ID for data submission", + "title": "Connection Id", + "type": "string" + }, + "timezone": { + "default": "UTC", + "description": "Timezone for date handling (default: UTC)", + "title": "Timezone", + "type": "string" + } + }, + "required": [ + "api_key", + "connection_id" + ], + "title": "CloudZeroInitRequest", + "type": "object" + }, + "CloudZeroInitResponse": { + "description": "Response model for CloudZero initialization", + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "title": "CloudZeroInitResponse", + "type": "object" + }, + "CloudZeroSettingsUpdate": { + "description": "Request model for updating CloudZero settings", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New CloudZero API key for authentication", + "title": "Api Key" + }, + "connection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New CloudZero connection ID for data submission", + "title": "Connection Id" + }, + "timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New timezone for date handling", + "title": "Timezone" + } + }, + "title": "CloudZeroSettingsUpdate", + "type": "object" + }, + "CloudZeroSettingsView": { + "description": "Response model for viewing CloudZero settings with masked API key", + "properties": { + "api_key_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked API key showing only first 4 and last 4 characters", + "title": "Api Key Masked" + }, + "connection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "CloudZero connection ID for data submission", + "title": "Connection Id" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configuration status", + "title": "Status" + }, + "timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Timezone for date handling", + "title": "Timezone" + } + }, + "title": "CloudZeroSettingsView", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/cloudzero/delete": { + "delete": { + "description": "Delete CloudZero settings from the database.\n\nThis endpoint removes the CloudZero configuration (API key, connection ID, timezone)\nfrom the proxy database. Only the CloudZero settings entry will be deleted;\nother configuration values in the database will remain unchanged.\n\nOnly admin users can delete CloudZero settings.", + "operationId": "delete_cloudzero_settings_cloudzero_delete_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/dry-run": { + "post": { + "description": "Perform a dry run export using the CloudZero logger.\n\nThis endpoint uses the CloudZero logger to perform a dry run export,\nwhich returns the data that would be exported without actually sending it to CloudZero.\n\nParameters:\n- limit: Optional limit on number of records to process (default: 10000)\n\nReturns:\n- usage_data: Sample of the raw usage data (first 50 records)\n- cbf_data: CloudZero CBF formatted data ready for export\n- summary: Statistics including total cost, tokens, and record counts\n\nOnly admin users can perform CloudZero exports.", + "operationId": "cloudzero_dry_run_export_cloudzero_dry_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cloudzero Dry Run Export", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/export": { + "post": { + "description": "Perform an actual export using the CloudZero logger.\n\nThis endpoint uses the CloudZero logger to export usage data to CloudZero AnyCost API.\n\nParameters:\n- limit: Optional limit on number of records to export\n- operation: CloudZero operation type (\"replace_hourly\" or \"sum\", default: \"replace_hourly\")\n\nOnly admin users can perform CloudZero exports.", + "operationId": "cloudzero_export_cloudzero_export_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cloudzero Export", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/init": { + "post": { + "description": "Initialize CloudZero settings and store in the database.\n\nThis endpoint stores the CloudZero API key, connection ID, and timezone configuration\nin the proxy database for use by the CloudZero logger.\n\nParameters:\n- api_key: CloudZero API key for authentication\n- connection_id: CloudZero connection ID for data submission\n- timezone: Timezone for date handling (default: UTC)\n\nOnly admin users can configure CloudZero settings.", + "operationId": "init_cloudzero_settings_cloudzero_init_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Init Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/settings": { + "get": { + "description": "View current CloudZero settings.\n\nReturns the current CloudZero configuration with the API key masked for security.\nOnly the first 4 and last 4 characters of the API key are shown.\nReturns null/empty values when settings are not configured (consistent with other settings endpoints).\n\nOnly admin users can view CloudZero settings.", + "operationId": "get_cloudzero_settings_cloudzero_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroSettingsView" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Cloudzero Settings", + "tags": [ + "cloudzero" + ] + }, + "put": { + "description": "Update existing CloudZero settings.\n\nAllows updating individual CloudZero configuration fields without requiring all fields.\nOnly provided fields will be updated; others will remain unchanged.\n\nParameters:\n- api_key: (Optional) New CloudZero API key for authentication\n- connection_id: (Optional) New CloudZero connection ID for data submission\n- timezone: (Optional) New timezone for date handling\n\nOnly admin users can update CloudZero settings.", + "operationId": "update_cloudzero_settings_cloudzero_settings_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroSettingsUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + } + } + }, + "compliance": { + "components": { + "schemas": { + "ComplianceCheckRequest": { + "description": "Request payload for compliance check endpoints.\n\nMirrors the spend log fields needed for compliance evaluation.", + "properties": { + "guardrail_information": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Guardrail Information" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "timestamp": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Timestamp" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + } + }, + "required": [ + "request_id" + ], + "title": "ComplianceCheckRequest", + "type": "object" + }, + "ComplianceCheckResult": { + "description": "Result of a single compliance check.", + "properties": { + "article": { + "title": "Article", + "type": "string" + }, + "check_name": { + "title": "Check Name", + "type": "string" + }, + "detail": { + "title": "Detail", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "boolean" + } + }, + "required": [ + "check_name", + "article", + "passed", + "detail" + ], + "title": "ComplianceCheckResult", + "type": "object" + }, + "ComplianceResponse": { + "description": "Response from a compliance check endpoint.", + "properties": { + "checks": { + "items": { + "$ref": "#/components/schemas/ComplianceCheckResult" + }, + "title": "Checks", + "type": "array" + }, + "compliant": { + "title": "Compliant", + "type": "boolean" + }, + "regulation": { + "title": "Regulation", + "type": "string" + } + }, + "required": [ + "compliant", + "regulation", + "checks" + ], + "title": "ComplianceResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/compliance/eu-ai-act": { + "post": { + "description": "Check EU AI Act compliance for a spend log entry.\n\nChecks:\n- Art. 9: Guardrails applied (any guardrail)\n- Art. 5: Content screened before LLM (pre-call guardrails)\n- Art. 12: Audit record complete (user_id, model, timestamp, guardrail_results)", + "operationId": "check_eu_ai_act_compliance_compliance_eu_ai_act_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceCheckRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Check Eu Ai Act Compliance", + "tags": [ + "compliance" + ] + } + }, + "/compliance/gdpr": { + "post": { + "description": "Check GDPR compliance for a spend log entry.\n\nChecks:\n- Art. 32: Data protection applied (pre-call guardrails)\n- Art. 5(1)(c): Sensitive data protected (masked/blocked or no issues)\n- Art. 30: Audit record complete (user_id, model, timestamp, guardrail_results)", + "operationId": "check_gdpr_compliance_compliance_gdpr_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceCheckRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Check Gdpr Compliance", + "tags": [ + "compliance" + ] + } + } + } + }, + "config_overrides": { + "components": { + "schemas": { + "ConfigOverrideSettingsResponse": { + "description": "Response model for config override settings GET endpoints.", + "properties": { + "config_type": { + "description": "The type of config override", + "title": "Config Type", + "type": "string" + }, + "field_schema": { + "additionalProperties": true, + "description": "Schema information for UI rendering", + "title": "Field Schema", + "type": "object" + }, + "values": { + "additionalProperties": true, + "description": "Current configuration values (sensitive fields decrypted)", + "title": "Values", + "type": "object" + } + }, + "required": [ + "config_type", + "values", + "field_schema" + ], + "title": "ConfigOverrideSettingsResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "HashicorpVaultConfig": { + "description": "Configuration for Hashicorp Vault secret manager integration.", + "properties": { + "approle_mount_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Mount path for the AppRole auth method (default: approle)", + "title": "Approle Mount Path" + }, + "approle_role_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Role ID for Vault AppRole authentication", + "title": "Approle Role Id" + }, + "approle_secret_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Secret ID for Vault AppRole authentication", + "title": "Approle Secret Id" + }, + "client_cert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to the client TLS certificate for Vault", + "title": "Client Cert" + }, + "client_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to the client TLS private key for Vault", + "title": "Client Key" + }, + "vault_addr": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The address of the Vault server (e.g., https://vault.example.com:8200)", + "title": "Vault Addr" + }, + "vault_cert_role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Certificate role name for TLS cert authentication", + "title": "Vault Cert Role" + }, + "vault_mount_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "KV engine mount name (default: secret)", + "title": "Vault Mount Name" + }, + "vault_namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Vault namespace (for multi-tenant Vault, sent as X-Vault-Namespace header)", + "title": "Vault Namespace" + }, + "vault_path_prefix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional path prefix for secrets (e.g., myapp -> secret/data/myapp/{secret_name})", + "title": "Vault Path Prefix" + }, + "vault_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Token for Vault token-based authentication", + "title": "Vault Token" + } + }, + "title": "HashicorpVaultConfig", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/config_overrides/hashicorp_vault": { + "delete": { + "description": "Delete Hashicorp Vault configuration. Idempotent.", + "operationId": "delete_hashicorp_vault_config_config_overrides_hashicorp_vault_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + }, + "get": { + "description": "Get current Hashicorp Vault configuration.\nReturns decrypted values from DB, or falls back to current env vars.", + "operationId": "get_hashicorp_vault_config_config_overrides_hashicorp_vault_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConfigOverrideSettingsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + }, + "post": { + "description": "Update Hashicorp Vault secret manager configuration.\nSets environment variables, encrypts sensitive fields, and stores in DB.\nReinitializes the secret manager on this pod.", + "operationId": "update_hashicorp_vault_config_config_overrides_hashicorp_vault_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HashicorpVaultConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + } + }, + "/config_overrides/hashicorp_vault/test_connection": { + "post": { + "description": "Test the connection to the currently configured Hashicorp Vault.\nUses the already-initialized secret manager client. Does not modify any state.", + "operationId": "test_hashicorp_vault_connection_config_overrides_hashicorp_vault_test_connection_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Hashicorp Vault Connection", + "tags": [ + "config_overrides" + ] + } + } + } + }, + "evals": { + "components": { + "schemas": { + "CancelEvalResponse": { + "description": "Response from cancelling an evaluation", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "object": { + "default": "eval", + "title": "Object", + "type": "string" + }, + "status": { + "const": "cancelled", + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "title": "CancelEvalResponse", + "type": "object" + }, + "CancelRunResponse": { + "description": "Response from cancelling a run", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "object": { + "default": "eval.run", + "title": "Object", + "type": "string" + }, + "status": { + "const": "cancelled", + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "title": "CancelRunResponse", + "type": "object" + }, + "DeleteEvalResponse": { + "description": "Response from deleting an evaluation", + "properties": { + "deleted": { + "title": "Deleted", + "type": "boolean" + }, + "eval_id": { + "title": "Eval Id", + "type": "string" + }, + "object": { + "default": "eval.deleted", + "title": "Object", + "type": "string" + } + }, + "required": [ + "eval_id", + "deleted" + ], + "title": "DeleteEvalResponse", + "type": "object" + }, + "Eval": { + "description": "Represents an evaluation from the OpenAI Evals API", + "properties": { + "created_at": { + "title": "Created At", + "type": "integer" + }, + "data_source_config": { + "additionalProperties": true, + "title": "Data Source Config", + "type": "object" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "object": { + "default": "eval", + "title": "Object", + "type": "string" + }, + "testing_criteria": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Testing Criteria", + "type": "array" + }, + "updated_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "id", + "created_at", + "data_source_config", + "testing_criteria" + ], + "title": "Eval", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListEvalsResponse": { + "description": "Response from listing evaluations", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Eval" + }, + "title": "Data", + "type": "array" + }, + "first_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "First Id" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "last_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Id" + }, + "object": { + "default": "list", + "title": "Object", + "type": "string" + } + }, + "required": [ + "data" + ], + "title": "ListEvalsResponse", + "type": "object" + }, + "ListRunsResponse": { + "description": "Response from listing runs", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Run" + }, + "title": "Data", + "type": "array" + }, + "first_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "First Id" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "last_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Id" + }, + "object": { + "default": "list", + "title": "Object", + "type": "string" + } + }, + "required": [ + "data" + ], + "title": "ListRunsResponse", + "type": "object" + }, + "PerTestingCriteriaResult": { + "description": "Results for a specific testing criteria", + "properties": { + "average_score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Score" + }, + "result_counts": { + "$ref": "#/components/schemas/ResultCounts" + }, + "testing_criteria_index": { + "title": "Testing Criteria Index", + "type": "integer" + } + }, + "required": [ + "testing_criteria_index", + "result_counts" + ], + "title": "PerTestingCriteriaResult", + "type": "object" + }, + "ResultCounts": { + "description": "Result counts for a run", + "properties": { + "error": { + "default": 0, + "title": "Error", + "type": "integer" + }, + "failed": { + "default": 0, + "title": "Failed", + "type": "integer" + }, + "passed": { + "default": 0, + "title": "Passed", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total" + ], + "title": "ResultCounts", + "type": "object" + }, + "Run": { + "description": "Represents a run from the OpenAI Evals API", + "properties": { + "completed_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Completed At" + }, + "created_at": { + "title": "Created At", + "type": "integer" + }, + "data_source": { + "additionalProperties": true, + "title": "Data Source", + "type": "object" + }, + "error": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "eval_id": { + "title": "Eval Id", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "object": { + "default": "eval.run", + "title": "Object", + "type": "string" + }, + "per_model_usage": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Per Model Usage" + }, + "per_testing_criteria_results": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PerTestingCriteriaResult" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Per Testing Criteria Results" + }, + "report_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Report Url" + }, + "result_counts": { + "anyOf": [ + { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Result Counts" + }, + "shared_with_openai": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Shared With Openai" + }, + "started_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Started At" + }, + "status": { + "enum": [ + "queued", + "running", + "completed", + "failed", + "cancelled" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "created_at", + "status", + "data_source", + "eval_id" + ], + "title": "Run", + "type": "object" + }, + "RunDeleteResponse": { + "description": "Response from deleting a run", + "properties": { + "deleted": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "title": "Deleted" + }, + "object": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "eval.run.deleted", + "title": "Object" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id" + ], + "title": "RunDeleteResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/evals": { + "get": { + "description": "List evaluations with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals?limit=10\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListEvalsResponse with list of evaluations", + "operationId": "list_evals_v1_evals_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order" + } + }, + { + "in": "query", + "name": "order_by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order By" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListEvalsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Evals", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Create a new evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\n \"name\": \"Test Eval\",\n \"data_source_config\": {\"type\": \"file\", \"file_id\": \"file-abc123\"},\n \"testing_criteria\": {\"graders\": [{\"type\": \"llm_as_judge\"}]}\n }'\n```\n\nReturns: Eval object with id, status, timestamps, etc.", + "operationId": "create_eval_v1_evals_post", + "parameters": [ + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}": { + "delete": { + "description": "Delete an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: DeleteEvalResponse with deletion confirmation", + "operationId": "delete_eval_v1_evals__eval_id__delete", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteEvalResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Eval", + "tags": [ + "evals" + ] + }, + "get": { + "description": "Get a specific evaluation by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Eval object", + "operationId": "get_eval_v1_evals__eval_id__get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Eval", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Update an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\"name\": \"Updated Name\"}'\n```\n\nReturns: Updated Eval object", + "operationId": "update_eval_v1_evals__eval_id__post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/cancel": { + "post": { + "description": "Cancel a running evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/cancel\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelEvalResponse with cancellation confirmation", + "operationId": "cancel_eval_v1_evals__eval_id__cancel_post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CancelEvalResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cancel Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/runs": { + "get": { + "description": "List all runs for an evaluation with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs?limit=10\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListRunsResponse with list of runs", + "operationId": "list_runs_v1_evals__eval_id__runs_get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListRunsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Runs", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Create a new run for an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n- Pass model via completion.model: `{\"completion\": {\"model\": \"gpt-4-account-1\"}}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\n \"data_source\": {\"type\": \"dataset\", \"dataset_id\": \"dataset_123\"},\n \"completion\": {\"model\": \"gpt-4\", \"temperature\": 0.7}\n }'\n```\n\nReturns: Run object with id, status, timestamps, etc.", + "operationId": "create_run_v1_evals__eval_id__runs_post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Run" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Run", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/runs/{run_id}": { + "delete": { + "description": "Delete a run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123/runs/run_456\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: RunDeleteResponse with deletion confirmation", + "operationId": "delete_run_v1_evals__eval_id__runs__run_id__delete", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RunDeleteResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Run", + "tags": [ + "evals" + ] + }, + "get": { + "description": "Get a specific run by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs/run_456\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Run object with full details", + "operationId": "get_run_v1_evals__eval_id__runs__run_id__get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Run" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Run", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Cancel a running run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelRunResponse with cancellation confirmation", + "operationId": "cancel_run_v1_evals__eval_id__runs__run_id__post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CancelRunResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cancel Run", + "tags": [ + "evals" + ] + } + } + } + }, + "guardrails": { + "components": { + "schemas": { + "ApplyGuardrailRequest": { + "properties": { + "entities": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PiiEntityType" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Entities" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "input_type": { + "default": "request", + "title": "Input Type", + "type": "string" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "messages": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Messages" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "guardrail_name", + "text" + ], + "title": "ApplyGuardrailRequest", + "type": "object" + }, + "ApplyGuardrailResponse": { + "properties": { + "response_text": { + "title": "Response Text", + "type": "string" + } + }, + "required": [ + "response_text" + ], + "title": "ApplyGuardrailResponse", + "type": "object" + }, + "BaseLitellmParams-Input": { + "additionalProperties": true, + "properties": { + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the guardrail service API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the guardrail service", + "title": "Api Key" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the guardrail in guardrails.ai", + "title": "Guard Name" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "Behavior when a guardrail endpoint is unreachable due to network errors. NOTE: This is currently only implemented by guardrail='generic_guardrail_api'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + } + }, + "title": "BaseLitellmParams", + "type": "object" + }, + "BaseLitellmParams-Output": { + "additionalProperties": true, + "properties": { + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the guardrail service API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the guardrail service", + "title": "Api Key" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the guardrail in guardrails.ai", + "title": "Guard Name" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "Behavior when a guardrail endpoint is unreachable due to network errors. NOTE: This is currently only implemented by guardrail='generic_guardrail_api'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + } + }, + "title": "BaseLitellmParams", + "type": "object" + }, + "BlockedWord": { + "description": "Represents a blocked word with its action and optional description", + "properties": { + "action": { + "$ref": "#/components/schemas/ContentFilterAction", + "description": "Action to take when keyword is detected (BLOCK or MASK)" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional description explaining why this keyword is sensitive", + "title": "Description" + }, + "keyword": { + "description": "The keyword to block or mask", + "title": "Keyword", + "type": "string" + } + }, + "required": [ + "keyword", + "action" + ], + "title": "BlockedWord", + "type": "object" + }, + "ContentFilterAction": { + "description": "Action to take when content filter detects a match", + "enum": [ + "BLOCK", + "MASK" + ], + "title": "ContentFilterAction", + "type": "string" + }, + "ContentFilterCategoryConfig": { + "additionalProperties": true, + "description": "category: \"harmful_self_harm\"\n enabled: true\n action: \"BLOCK\"\n severity_threshold: \"medium\"\n category_file: \"/path/to/custom_file.yaml\" # optional override", + "properties": { + "action": { + "description": "The action to take when the category is detected", + "enum": [ + "BLOCK", + "MASK" + ], + "title": "Action", + "type": "string" + }, + "category": { + "description": "The category to detect", + "title": "Category", + "type": "string" + }, + "category_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional override. Use your own category file instead of the default one.", + "title": "Category File" + }, + "enabled": { + "default": true, + "description": "Whether the category is enabled", + "title": "Enabled", + "type": "boolean" + }, + "severity_threshold": { + "default": "medium", + "description": "The severity threshold to detect the category", + "enum": [ + "high", + "medium", + "low" + ], + "title": "Severity Threshold", + "type": "string" + } + }, + "required": [ + "category", + "action" + ], + "title": "ContentFilterCategoryConfig", + "type": "object" + }, + "ContentFilterPattern": { + "description": "Represents a content filter pattern (prebuilt or custom regex)", + "properties": { + "action": { + "$ref": "#/components/schemas/ContentFilterAction", + "description": "Action to take when pattern matches (BLOCK or MASK)" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name for this pattern (used in logging and error messages)", + "title": "Name" + }, + "pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom regex pattern. Required if pattern_type is 'regex'", + "title": "Pattern" + }, + "pattern_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of prebuilt pattern (e.g., 'us_ssn', 'credit_card'). Required if pattern_type is 'prebuilt'", + "title": "Pattern Name" + }, + "pattern_type": { + "description": "Type of pattern: 'prebuilt' for predefined patterns or 'regex' for custom", + "enum": [ + "prebuilt", + "regex" + ], + "title": "Pattern Type", + "type": "string" + } + }, + "required": [ + "pattern_type", + "action" + ], + "title": "ContentFilterPattern", + "type": "object" + }, + "CreateGuardrailRequest": { + "properties": { + "guardrail": { + "$ref": "#/components/schemas/Guardrail" + } + }, + "required": [ + "guardrail" + ], + "title": "CreateGuardrailRequest", + "type": "object" + }, + "GUARDRAIL_DEFINITION_LOCATION": { + "enum": [ + "db", + "config" + ], + "title": "GUARDRAIL_DEFINITION_LOCATION", + "type": "string" + }, + "GraySwanGuardrailConfigModelOptionalParams": { + "description": "Optional parameters for the Gray Swan guardrail.", + "properties": { + "categories": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Default Gray Swan category definitions to send with each request.", + "title": "Categories" + }, + "fail_open": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "If true (default), errors contacting Gray Swan are logged and the request proceeds. If false, errors propagate and block the request.", + "title": "Fail Open" + }, + "guardrail_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": 30.0, + "description": "Timeout in seconds for calling the Gray Swan guardrail service.", + "title": "Guardrail Timeout" + }, + "on_flagged_action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "passthrough", + "description": "Action when a violation is detected: 'block' rejects the call (400 error), 'monitor' logs only, 'passthrough' replaces response content with violation message (200 status).", + "title": "On Flagged Action" + }, + "policy_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Gray Swan policy identifier to apply during monitoring.", + "title": "Policy Id" + }, + "reasoning_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Gray Swan reasoning mode override. Accepted values: 'off', 'hybrid', 'thinking'.", + "title": "Reasoning Mode" + }, + "violation_threshold": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": 0.5, + "description": "Threshold between 0 and 1 at which Gray Swan violations trigger the configured action.", + "title": "Violation Threshold" + } + }, + "title": "GraySwanGuardrailConfigModelOptionalParams", + "type": "object" + }, + "Guardrail": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "$ref": "#/components/schemas/LitellmParams" + }, + "policy_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Policy Template" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_name", + "litellm_params" + ], + "title": "Guardrail", + "type": "object" + }, + "GuardrailInfoResponse": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_definition_location": { + "$ref": "#/components/schemas/GUARDRAIL_DEFINITION_LOCATION", + "default": "config" + }, + "guardrail_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/BaseLitellmParams-Output" + }, + { + "type": "null" + } + ] + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_name" + ], + "title": "GuardrailInfoResponse", + "type": "object" + }, + "GuardrailSubmissionItem": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "reviewed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed At" + }, + "status": { + "title": "Status", + "type": "string" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + }, + "submitted_by_email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By Email" + }, + "submitted_by_user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By User Id" + }, + "team_guardrail": { + "default": false, + "title": "Team Guardrail", + "type": "boolean" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "status" + ], + "title": "GuardrailSubmissionItem", + "type": "object" + }, + "GuardrailSubmissionSummary": { + "properties": { + "active": { + "title": "Active", + "type": "integer" + }, + "pending_review": { + "title": "Pending Review", + "type": "integer" + }, + "rejected": { + "title": "Rejected", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total", + "pending_review", + "active", + "rejected" + ], + "title": "GuardrailSubmissionSummary", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LakeraCategoryThresholds": { + "additionalProperties": true, + "properties": { + "jailbreak": { + "title": "Jailbreak", + "type": "number" + }, + "prompt_injection": { + "title": "Prompt Injection", + "type": "number" + } + }, + "title": "LakeraCategoryThresholds", + "type": "object" + }, + "ListGuardrailSubmissionsResponse": { + "properties": { + "submissions": { + "items": { + "$ref": "#/components/schemas/GuardrailSubmissionItem" + }, + "title": "Submissions", + "type": "array" + }, + "summary": { + "$ref": "#/components/schemas/GuardrailSubmissionSummary" + } + }, + "required": [ + "submissions", + "summary" + ], + "title": "ListGuardrailSubmissionsResponse", + "type": "object" + }, + "ListGuardrailsResponse": { + "properties": { + "guardrails": { + "items": { + "$ref": "#/components/schemas/GuardrailInfoResponse" + }, + "title": "Guardrails", + "type": "array" + } + }, + "required": [ + "guardrails" + ], + "title": "ListGuardrailsResponse", + "type": "object" + }, + "LitellmParams": { + "additionalProperties": true, + "properties": { + "action": { + "default": "block", + "description": "'block' raises an error; 'mask' replaces the code block with a placeholder.", + "enum": [ + "block", + "mask" + ], + "title": "Action", + "type": "string" + }, + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "akto_account_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto account ID for multi-tenant deployments. Env: AKTO_ACCOUNT_ID. Default: '1000000'.", + "title": "Akto Account Id" + }, + "akto_api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for Akto. Env: AKTO_API_KEY.", + "title": "Akto Api Key" + }, + "akto_base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto Guardrail API Base URL. Env: AKTO_GUARDRAIL_API_BASE.", + "examples": [ + "http://localhost:9090", + "https://akto-ingestion.example.com" + ], + "title": "Akto Base Url" + }, + "akto_vxlan_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto VXLAN ID. Env: AKTO_VXLAN_ID. Default: '0'.", + "title": "Akto Vxlan Id" + }, + "anonymize_input": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, replaces sensitive content with anonymized version when only PII/PCI/secrets are detected. Only applies in blocking mode. Defaults to False if not provided", + "title": "Anonymize Input" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Lakera AI API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The Hiddenlayer API Id for the Hiddenlayer API. If not provided, the `HIDDENLAYER_CLIENT_ID` environment variable is checked or https://api.hiddenlayer.ai is used.", + "title": "Api Id" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the Lakera AI service", + "title": "Api Key" + }, + "api_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "v1", + "description": "API version for Javelin service", + "title": "Api Version" + }, + "application": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Application name for Javelin service", + "title": "Application" + }, + "application_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Application ID for Noma Security. Defaults to 'litellm' if not provided", + "title": "Application Id" + }, + "assertions": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Custom assertions to validate against the output. Each assertion is a string describing a condition.", + "title": "Assertions" + }, + "async_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Set to True to request asynchronous analysis (sets `plr_async` header). Defaults to provider behaviour when omitted.", + "title": "Async Mode" + }, + "auth_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None.", + "title": "Auth Token" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS access key ID for authentication", + "title": "Aws Access Key Id" + }, + "aws_bedrock_runtime_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS Bedrock runtime endpoint URL", + "title": "Aws Bedrock Runtime Endpoint" + }, + "aws_profile_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS profile name for credential retrieval", + "title": "Aws Profile Name" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS region where your guardrail is deployed", + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS role name for assuming roles", + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS secret access key for authentication", + "title": "Aws Secret Access Key" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the AWS session", + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS session token for temporary credentials", + "title": "Aws Session Token" + }, + "aws_sts_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS STS endpoint URL", + "title": "Aws Sts Endpoint" + }, + "aws_web_identity_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Web identity token for AWS role assumption", + "title": "Aws Web Identity Token" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the IBM Guardrails server", + "title": "Base Url" + }, + "block_failures": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, blocks requests on API failures. Defaults to True if not provided", + "title": "Block Failures" + }, + "block_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether to block the request when the PromptGuard API is unreachable. Defaults to true (fail-closed). Set to false for fail-open behaviour.", + "title": "Block On Error" + }, + "block_on_violation": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to block requests when violations are detected. Defaults to True.", + "title": "Block On Violation" + }, + "blocked_languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Language tags to block (e.g. python, javascript, bash). Empty or None = block all fenced code blocks.", + "options": [ + "python", + "javascript", + "typescript", + "bash", + "ruby", + "go", + "java", + "csharp", + "php", + "c", + "cpp", + "rust", + "sql" + ], + "title": "Blocked Languages", + "ui_type": "multiselect" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "breakdown": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include breakdown in the response", + "title": "Breakdown" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "confidence_threshold": { + "default": 0.5, + "default_value": 0.5, + "description": "Only block or mask when detection confidence >= this value; below threshold, allow or log_only.", + "max": 1.0, + "maximum": 1.0, + "min": 0.0, + "minimum": 0.0, + "step": 0.1, + "title": "Confidence Threshold", + "type": "number", + "ui_type": "percentage" + }, + "config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional configuration for the guardrail", + "title": "Config" + }, + "content_moderation_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable content moderation to check for harmful content (harassment, hate speech, etc.).", + "title": "Content Moderation Check" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_action": { + "default": "deny", + "description": "Fallback decision when no rule matches", + "enum": [ + "allow", + "deny" + ], + "title": "Default Action", + "type": "string" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "deployment_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The EnkryptAI deployment name to use. Sent via X-Enkrypt-Deployment header.", + "title": "Deployment Name" + }, + "detect_execution_intent": { + "default": true, + "description": "When True, block only when user intent is to run/execute; allow when intent is explain/refactor/don't run. Also block text-only execution requests (e.g. 'run `ls`', 'read /etc/passwd').", + "title": "Detect Execution Intent", + "type": "boolean" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "detector_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the detector inside the server (e.g., 'jailbreak-detector')", + "title": "Detector Id" + }, + "detectors": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dictionary of detector configurations (e.g., {'nsfw': {'enabled': True}, 'toxicity': {'enabled': True}}).", + "title": "Detectors" + }, + "dev_info": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include developer information in the response", + "title": "Dev Info" + }, + "disable_exception_on_block": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If True, will not raise an exception when the guardrail is blocked. Useful for OpenWebUI where exceptions can end the chat flow.", + "title": "Disable Exception On Block" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "evaluation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Pre-configured evaluation ID from Qualifire dashboard. When provided, uses invoke_evaluation() instead of evaluate().", + "title": "Evaluation Id" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "grounding_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable grounding verification to ensure output is grounded in provided context.", + "title": "Grounding Check" + }, + "grounding_strictness": { + "anyOf": [ + { + "enum": [ + "BALANCED", + "STRICT" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Strictness level for XecGuard context-grounding validation. 'BALANCED' (default) treats INCOMPLETE answers as SAFE; 'STRICT' flags them as UNSAFE. Grounding only runs in post_call when `metadata.xecguard_grounding_documents` is provided.", + "title": "Grounding Strictness" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the Javelin guard to use", + "title": "Guard Name" + }, + "guardrail": { + "description": "The type of guardrail integration to use", + "title": "Guardrail", + "type": "string" + }, + "guardrailIdentifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your guardrail on Bedrock", + "title": "Guardrailidentifier" + }, + "guardrailVersion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The version of your Bedrock guardrail (e.g., DRAFT or version number)", + "title": "Guardrailversion" + }, + "guardrail_timeout": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "HTTP timeout in seconds. Default: 5.", + "title": "Guardrail Timeout" + }, + "hallucinations_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable hallucination detection to detect factual inaccuracies.", + "title": "Hallucinations Check" + }, + "include_evidence": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Include detailed evidence payloads in responses (sets `plr_evidence` header).", + "title": "Include Evidence" + }, + "include_scanners": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Include scanner category summaries in responses (sets `plr_scanners` header).", + "title": "Include Scanners" + }, + "is_detector_server": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Boolean flag to determine if calling a detector server (True) or the FMS Orchestrator (False). Defaults to True.", + "title": "Is Detector Server" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "lasso_conversation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Conversation ID for the Lasso guardrail", + "title": "Lasso Conversation Id" + }, + "lasso_user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "User ID for the Lasso guardrail", + "title": "Lasso User Id" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Enable content masking using Lasso classifix API", + "title": "Mask" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional metadata to include in the request", + "title": "Metadata" + }, + "mock_redacted_text": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Mock redacted text for testing", + "title": "Mock Redacted Text" + }, + "mode": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/components/schemas/Mode" + } + ], + "description": "When to apply the guardrail (pre_call, post_call, during_call, logging_only)", + "title": "Mode" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "monitor_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, logs violations without blocking. Defaults to False if not provided", + "title": "Monitor Mode" + }, + "on_disallowed_action": { + "default": "block", + "description": "Choose whether disallowed tools block the request or get rewritten out of the payload", + "enum": [ + "block", + "rewrite" + ], + "title": "On Disallowed Action", + "type": "string" + }, + "on_flagged": { + "anyOf": [ + { + "enum": [ + "block", + "monitor" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": "block", + "description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)", + "title": "On Flagged" + }, + "on_flagged_action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "monitor", + "description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)", + "title": "On Flagged Action" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "optional_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/GraySwanGuardrailConfigModelOptionalParams" + }, + { + "type": "null" + } + ], + "description": "Optional parameters for the guardrail" + }, + "output_parse_pii": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, LiteLLM will replace the masked text with the original text in the response", + "title": "Output Parse Pii", + "ui_type": "bool" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "payload": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include payload in the response", + "title": "Payload" + }, + "persist_session": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Controls Pillar session persistence (sets `plr_persist` header). Set to False to disable persistence.", + "title": "Persist Session" + }, + "pii_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable PII (Personally Identifiable Information) detection.", + "title": "Pii Check" + }, + "pii_entities_config": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/components/schemas/PiiAction" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for PII entity types and actions", + "title": "Pii Entities Config" + }, + "policy_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Policy ID for Zscaler AI Guard. Can also be set via ZSCALER_AI_GUARD_POLICY_ID environment variable", + "title": "Policy Id" + }, + "policy_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The EnkryptAI policy name to use. Sent via x-enkrypt-policy header.", + "title": "Policy Name" + }, + "policy_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "XecGuard policies to apply on each scan. Select one or more of the built-in default policies; if none are selected, the guardrail defaults to System Prompt Enforcement + Harmful Content Protection.", + "options": [ + "Default_Policy_SystemPromptEnforcement", + "Default_Policy_GeneralPromptAttackProtection", + "Default_Policy_ContentBiasProtection", + "Default_Policy_HarmfulContentProtection", + "Default_Policy_SkillsProtection", + "Default_Policy_PIISensitiveDataProtection" + ], + "title": "Policy Names", + "ui_type": "multiselect" + }, + "presidio_ad_hoc_recognizers": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to a JSON file containing ad-hoc recognizers for Presidio", + "title": "Presidio Ad Hoc Recognizers" + }, + "presidio_analyzer_api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Presidio analyzer API", + "title": "Presidio Analyzer Api Base" + }, + "presidio_anonymizer_api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Presidio anonymizer API", + "title": "Presidio Anonymizer Api Base" + }, + "presidio_entities_deny_list": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/PiiEntityType" + }, + { + "type": "string" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of entity types to exclude from Presidio detection results. Detections of these types will be silently dropped. Useful for suppressing false positives (e.g., US_DRIVER_LICENSE on coding routes).", + "title": "Presidio Entities Deny List" + }, + "presidio_filter_scope": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Where to apply Presidio checks: 'input' (user -> model), 'output' (model -> user), or 'both' (default).", + "title": "Presidio Filter Scope" + }, + "presidio_language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "en", + "description": "Language code for Presidio PII analysis (e.g., 'en', 'de', 'es', 'fr')", + "title": "Presidio Language" + }, + "presidio_run_on": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Where to apply Presidio checks: input, output, or both (default).", + "title": "Presidio Run On" + }, + "presidio_score_thresholds": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional per-entity minimum confidence scores for Presidio detections. Entities below the threshold are ignored.", + "title": "Presidio Score Thresholds" + }, + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Project ID for the Lakera AI project", + "title": "Project Id" + }, + "prompt_injections": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable prompt injection detection. Default check if no evaluation_id and no other checks are specified.", + "title": "Prompt Injections" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "rules": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ToolPermissionRule" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Ordered allow/deny rules. Patterns use regex for tool names/types and optional regex constraints on tool arguments.", + "title": "Rules" + }, + "send_user_api_key_alias": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_alias in headers", + "title": "Send User Api Key Alias" + }, + "send_user_api_key_team_id": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_team_id in headers", + "title": "Send User Api Key Team Id" + }, + "send_user_api_key_user_id": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_user_id in headers", + "title": "Send User Api Key User Id" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "tool_selection_quality_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable tool selection quality check to evaluate quality of tool/function calls.", + "title": "Tool Selection Quality Check" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "What to do when Akto is unreachable. 'fail_open' = allow, 'fail_closed' = block.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "use_v2": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If True and guardrail='noma', route to the new Noma v2 implementation instead of the legacy implementation.", + "title": "Use V2" + }, + "verify_ssl": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to verify SSL certificates. Defaults to True.", + "title": "Verify Ssl" + }, + "version": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 2, + "description": "Hiddenlayer guardrail version to use.", + "title": "Version" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + }, + "xecguard_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "XecGuard scanning model identifier. Defaults to 'xecguard_v2'.", + "title": "Xecguard Model" + } + }, + "required": [ + "guardrail", + "mode" + ], + "title": "LitellmParams", + "type": "object" + }, + "Mode": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Default mode when no tags match", + "title": "Default" + }, + "tags": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "description": "Tags for the guardrail mode", + "title": "Tags", + "type": "object" + } + }, + "required": [ + "tags" + ], + "title": "Mode", + "type": "object" + }, + "PatchGuardrailRequest": { + "properties": { + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Name" + }, + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/BaseLitellmParams-Input" + }, + { + "type": "null" + } + ] + } + }, + "title": "PatchGuardrailRequest", + "type": "object" + }, + "PiiAction": { + "enum": [ + "BLOCK", + "MASK" + ], + "title": "PiiAction", + "type": "string" + }, + "PiiEntityType": { + "enum": [ + "CREDIT_CARD", + "CRYPTO", + "DATE_TIME", + "EMAIL_ADDRESS", + "IBAN_CODE", + "IP_ADDRESS", + "NRP", + "LOCATION", + "PERSON", + "PHONE_NUMBER", + "MEDICAL_LICENSE", + "URL", + "US_BANK_NUMBER", + "US_DRIVER_LICENSE", + "US_ITIN", + "US_PASSPORT", + "US_SSN", + "UK_NHS", + "UK_NINO", + "ES_NIF", + "ES_NIE", + "IT_FISCAL_CODE", + "IT_DRIVER_LICENSE", + "IT_VAT_CODE", + "IT_PASSPORT", + "IT_IDENTITY_CARD", + "PL_PESEL", + "SG_NRIC_FIN", + "SG_UEN", + "AU_ABN", + "AU_ACN", + "AU_TFN", + "AU_MEDICARE", + "IN_PAN", + "IN_AADHAAR", + "IN_VEHICLE_REGISTRATION", + "IN_VOTER", + "IN_PASSPORT", + "FI_PERSONAL_IDENTITY_CODE" + ], + "title": "PiiEntityType", + "type": "string" + }, + "RegisterGuardrailRequest": { + "description": "Request body for POST /guardrails/register. Follows Generic Guardrail API config.", + "properties": { + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + "required": [ + "guardrail_name", + "litellm_params" + ], + "title": "RegisterGuardrailRequest", + "type": "object" + }, + "RegisterGuardrailResponse": { + "properties": { + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "status" + ], + "title": "RegisterGuardrailResponse", + "type": "object" + }, + "TestCustomCodeGuardrailRequest": { + "description": "Request model for testing custom code guardrails.", + "properties": { + "custom_code": { + "title": "Custom Code", + "type": "string" + }, + "input_type": { + "default": "request", + "title": "Input Type", + "type": "string" + }, + "request_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Request Data" + }, + "test_input": { + "additionalProperties": true, + "title": "Test Input", + "type": "object" + } + }, + "required": [ + "custom_code", + "test_input" + ], + "title": "TestCustomCodeGuardrailRequest", + "type": "object" + }, + "TestCustomCodeGuardrailResponse": { + "description": "Response model for testing custom code guardrails.", + "properties": { + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "error_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error Type" + }, + "result": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Result" + }, + "success": { + "title": "Success", + "type": "boolean" + } + }, + "required": [ + "success" + ], + "title": "TestCustomCodeGuardrailResponse", + "type": "object" + }, + "ToolPermissionRule": { + "description": "A rule defining permission for a specific tool or tool pattern", + "properties": { + "allowed_param_patterns": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional regex map enforcing nested parameter values using dot/[] paths", + "title": "Allowed Param Patterns" + }, + "decision": { + "description": "Whether to allow or deny this tool usage", + "enum": [ + "allow", + "deny" + ], + "title": "Decision", + "type": "string" + }, + "id": { + "description": "Unique identifier for the rule", + "title": "Id", + "type": "string" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Regex pattern applied to the tool's function name", + "title": "Tool Name" + }, + "tool_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Regex pattern applied to the tool type (e.g., function)", + "title": "Tool Type" + } + }, + "required": [ + "id", + "decision" + ], + "title": "ToolPermissionRule", + "type": "object" + }, + "UpdateGuardrailRequest": { + "properties": { + "guardrail": { + "$ref": "#/components/schemas/Guardrail" + } + }, + "required": [ + "guardrail" + ], + "title": "UpdateGuardrailRequest", + "type": "object" + }, + "UsageDetailResponse": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "time_series": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Time Series", + "type": "array" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend", + "description", + "time_series" + ], + "title": "UsageDetailResponse", + "type": "object" + }, + "UsageLogEntry": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "input_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Snippet" + }, + "latency_ms": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Latency Ms" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Snippet" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "timestamp": { + "title": "Timestamp", + "type": "string" + } + }, + "required": [ + "id", + "timestamp", + "action", + "score", + "latency_ms", + "model", + "input_snippet", + "output_snippet", + "reason" + ], + "title": "UsageLogEntry", + "type": "object" + }, + "UsageLogsResponse": { + "properties": { + "logs": { + "items": { + "$ref": "#/components/schemas/UsageLogEntry" + }, + "title": "Logs", + "type": "array" + }, + "page": { + "title": "Page", + "type": "integer" + }, + "page_size": { + "title": "Page Size", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "logs", + "total", + "page", + "page_size" + ], + "title": "UsageLogsResponse", + "type": "object" + }, + "UsageOverviewResponse": { + "properties": { + "chart": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Chart", + "type": "array" + }, + "passRate": { + "title": "Passrate", + "type": "number" + }, + "rows": { + "items": { + "$ref": "#/components/schemas/UsageOverviewRow" + }, + "title": "Rows", + "type": "array" + }, + "totalBlocked": { + "title": "Totalblocked", + "type": "integer" + }, + "totalRequests": { + "title": "Totalrequests", + "type": "integer" + } + }, + "required": [ + "rows", + "chart", + "totalRequests", + "totalBlocked", + "passRate" + ], + "title": "UsageOverviewResponse", + "type": "object" + }, + "UsageOverviewRow": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend" + ], + "title": "UsageOverviewRow", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/apply_guardrail": { + "post": { + "description": "Apply a guardrail to text input and return the processed result.\n\nThis endpoint allows testing guardrails by applying them to custom text inputs.", + "operationId": "apply_guardrail_apply_guardrail_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Apply Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails": { + "post": { + "description": "Create a new guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/guardrails\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail\": {\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "create_guardrail_guardrails_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/apply_guardrail": { + "post": { + "description": "Apply a guardrail to text input and return the processed result.\n\nThis endpoint allows testing guardrails by applying them to custom text inputs.", + "operationId": "apply_guardrail_guardrails_apply_guardrail_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Apply Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/list": { + "get": { + "description": "List the guardrails that are available on the proxy server\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrails\": [\n {\n \"guardrail_name\": \"bedrock-pre-guard\",\n \"guardrail_info\": {\n \"params\": [\n {\n \"name\": \"toxicity_score\",\n \"type\": \"float\",\n \"description\": \"Score between 0-1 indicating content toxicity level\"\n },\n {\n \"name\": \"pii_detection\",\n \"type\": \"boolean\"\n }\n ]\n }\n }\n ]\n}\n```", + "operationId": "list_guardrails_guardrails_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrails", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/register": { + "post": { + "description": "Register a guardrail for onboarding (team submission).\n\nAccepts a guardrail config in the\n[Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api) format.\nThe submission is stored with status `pending_review` until an admin approves it.", + "operationId": "register_guardrail_guardrails_register_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions": { + "get": { + "description": "List team guardrail submissions. Returns only guardrails with a team_id.\n\nAdmins see all submissions. Non-admin users see submissions for teams they are\na member of.\n\nStatus values: pending_review (team-registered, awaiting approval), active (approved), rejected.\n\nOptional filters:\n- status: pending_review | active | rejected\n- team_id: filter by specific team (non-admins must be a member of that team)\n- search: name/description", + "operationId": "list_guardrail_submissions_guardrails_submissions_get", + "parameters": [ + { + "in": "query", + "name": "status", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + }, + { + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + { + "in": "query", + "name": "search", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailSubmissionsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrail Submissions", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}": { + "get": { + "description": "Get a single guardrail submission by id. Non-admins may only access submissions for teams they belong to.", + "operationId": "get_guardrail_submission_guardrails_submissions__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GuardrailSubmissionItem" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}/approve": { + "post": { + "description": "Approve a pending guardrail submission: set status to active and initialize in memory (admin only).", + "operationId": "approve_guardrail_submission_guardrails_submissions__guardrail_id__approve_post", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Approve Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}/reject": { + "post": { + "description": "Reject a guardrail submission (admin only).", + "operationId": "reject_guardrail_submission_guardrails_submissions__guardrail_id__reject_post", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Reject Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/test_custom_code": { + "post": { + "description": "Test custom code guardrail logic without creating a guardrail.\n\nThis endpoint allows admins to experiment with custom code guardrails by:\n1. Compiling the provided code in a sandbox\n2. Executing the apply_guardrail function with test input\n3. Returning the result (allow/block/modify)\n\n\ud83d\udc49 [Custom Code Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/custom_code_guardrail)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/guardrails/test_custom_code\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"custom_code\": \"def apply_guardrail(inputs, request_data, input_type):\\n for text in inputs[\\\"texts\\\"]:\\n if regex_match(text, r\\\"\\\\d{3}-\\\\d{2}-\\\\d{4}\\\"):\\n return block(\\\"SSN detected\\\")\\n return allow()\",\n \"test_input\": {\n \"texts\": [\"My SSN is 123-45-6789\"]\n },\n \"input_type\": \"request\"\n }'\n```\n\nExample Success Response (blocked):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"block\",\n \"reason\": \"SSN detected\"\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Success Response (allowed):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"allow\"\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Success Response (modified):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"modify\",\n \"texts\": [\"My SSN is [REDACTED]\"]\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Error Response (compilation error):\n```json\n{\n \"success\": false,\n \"result\": null,\n \"error\": \"Syntax error in custom code: invalid syntax (, line 1)\",\n \"error_type\": \"compilation\"\n}\n```", + "operationId": "test_custom_code_guardrail_guardrails_test_custom_code_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestCustomCodeGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestCustomCodeGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Custom Code Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/add_guardrail_settings": { + "get": { + "description": "Get the UI settings for the guardrails\n\nReturns:\n- Supported entities for guardrails\n- Supported modes for guardrails\n- PII entity categories for UI organization\n- Content filter settings (patterns and categories)", + "operationId": "get_guardrail_ui_settings_guardrails_ui_add_guardrail_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Ui Settings", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/category_yaml/{category_name}": { + "get": { + "description": "Get the YAML or JSON content for a specific content filter category.\n\nArgs:\n category_name: The name of the category (e.g., \"bias_gender\", \"harmful_self_harm\")\n\nReturns:\n The raw YAML or JSON content of the category file with file type indicator", + "operationId": "get_category_yaml_guardrails_ui_category_yaml__category_name__get", + "parameters": [ + { + "in": "path", + "name": "category_name", + "required": true, + "schema": { + "title": "Category Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Category Yaml", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/major_airlines": { + "get": { + "description": "Get the major airlines list from IATA (competitor intent, airline type).\nReturns airline id, match variants (pipe-separated), and tags.", + "operationId": "get_major_airlines_guardrails_ui_major_airlines_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Major Airlines", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/provider_specific_params": { + "get": { + "description": "Get provider-specific parameters for different guardrail types.\n\nReturns a dictionary mapping guardrail providers to their specific parameters,\nincluding parameter names, descriptions, and whether they are required.\n\nExample Response:\n```json\n{\n \"bedrock\": {\n \"guardrailIdentifier\": {\n \"description\": \"The ID of your guardrail on Bedrock\",\n \"required\": true,\n \"type\": null\n },\n \"guardrailVersion\": {\n \"description\": \"The version of your Bedrock guardrail (e.g., DRAFT or version number)\",\n \"required\": true,\n \"type\": null\n }\n },\n \"azure_content_safety_text_moderation\": {\n \"api_key\": {\n \"description\": \"API key for the Azure Content Safety Text Moderation guardrail\",\n \"required\": false,\n \"type\": null\n },\n \"optional_params\": {\n \"description\": \"Optional parameters for the Azure Content Safety Text Moderation guardrail\",\n \"required\": true,\n \"type\": \"nested\",\n \"fields\": {\n \"severity_threshold\": {\n \"description\": \"Severity threshold for the Azure Content Safety Text Moderation guardrail across all categories\",\n \"required\": false,\n \"type\": null\n },\n \"categories\": {\n \"description\": \"Categories to scan for the Azure Content Safety Text Moderation guardrail\",\n \"required\": false,\n \"type\": \"multiselect\",\n \"options\": [\"Hate\", \"SelfHarm\", \"Sexual\", \"Violence\"],\n \"default_value\": None\n }\n }\n }\n }\n}\n```", + "operationId": "get_provider_specific_params_guardrails_ui_provider_specific_params_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Provider Specific Params", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/detail/{guardrail_id}": { + "get": { + "description": "Return single guardrail usage metrics and time series.", + "operationId": "guardrails_usage_detail_guardrails_usage_detail__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageDetailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Detail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/logs": { + "get": { + "description": "Return paginated run logs for a guardrail (or policy) from SpendLogs via index.", + "operationId": "guardrails_usage_logs_guardrails_usage_logs_get", + "parameters": [ + { + "in": "query", + "name": "guardrail_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + } + }, + { + "in": "query", + "name": "policy_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Policy Id" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "title": "Page Size", + "type": "integer" + } + }, + { + "in": "query", + "name": "action", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Action" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageLogsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Logs", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/overview": { + "get": { + "description": "Return guardrail performance overview for the dashboard.", + "operationId": "guardrails_usage_overview_guardrails_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Overview", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/validate_blocked_words_file": { + "post": { + "description": "Validate a blocked_words YAML file content.\n\nArgs:\n request: Dictionary with 'file_content' key containing the YAML string\n\nReturns:\n Dictionary with 'valid' boolean and either 'message'/'errors' depending on result\n\nExample Request:\n```json\n{\n \"file_content\": \"blocked_words:\\n - keyword: \\\"test\\\"\\n action: \\\"BLOCK\\\"\"\n}\n```\n\nExample Success Response:\n```json\n{\n \"valid\": true,\n \"message\": \"Valid YAML file with 2 blocked words\"\n}\n```\n\nExample Error Response:\n```json\n{\n \"valid\": false,\n \"errors\": [\"Entry 0: missing 'action' field\"]\n}\n```", + "operationId": "validate_blocked_words_file_guardrails_validate_blocked_words_file_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Request", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Validate Blocked Words File", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/{guardrail_id}": { + "delete": { + "description": "Delete a guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Guardrail 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_guardrail_guardrails__guardrail_id__delete", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Guardrail", + "tags": [ + "guardrails" + ] + }, + "get": { + "description": "Get detailed information about a specific guardrail by ID\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000/info\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_guardrail_info_guardrails__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Info", + "tags": [ + "guardrails" + ] + }, + "patch": { + "description": "Partially update an existing guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nThis endpoint allows updating specific fields of a guardrail without sending the entire object.\nOnly the following fields can be updated:\n- guardrail_name: The name of the guardrail\n- default_on: Whether the guardrail is enabled by default\n- guardrail_info: Additional information about the guardrail\n\nExample Request:\n```bash\ncurl -X PATCH \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail_name\": \"updated-name\",\n \"default_on\": true,\n \"guardrail_info\": {\n \"description\": \"Updated description\"\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"updated-name\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated description\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T14:22:33.456Z\"\n}\n```", + "operationId": "patch_guardrail_guardrails__guardrail_id__patch", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Guardrail", + "tags": [ + "guardrails" + ] + }, + "put": { + "description": "Update an existing guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail\": {\n \"guardrail_name\": \"updated-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"1.0\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated Bedrock content moderation guardrail\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"updated-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"1.0\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n}\n```", + "operationId": "update_guardrail_guardrails__guardrail_id__put", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/{guardrail_id}/info": { + "get": { + "description": "Get detailed information about a specific guardrail by ID\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000/info\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_guardrail_info_guardrails__guardrail_id__info_get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Info", + "tags": [ + "guardrails" + ] + } + }, + "/policies/usage/overview": { + "get": { + "description": "Return policy performance overview for the dashboard.", + "operationId": "policies_usage_overview_policies_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Policies Usage Overview", + "tags": [ + "guardrails" + ] + } + }, + "/v2/guardrails/list": { + "get": { + "description": "List the guardrails that are available in the database using GuardrailRegistry\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/v2/guardrails/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrails\": [\n {\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n }\n }\n ]\n}\n```", + "operationId": "list_guardrails_v2_v2_guardrails_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrails V2", + "tags": [ + "guardrails" + ] + } + } + } + }, + "jwt_mappings": { + "components": { + "schemas": { + "CreateJWTKeyMappingRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "jwt_claim_name": { + "title": "Jwt Claim Name", + "type": "string" + }, + "jwt_claim_value": { + "title": "Jwt Claim Value", + "type": "string" + }, + "key": { + "title": "Key", + "type": "string" + } + }, + "required": [ + "jwt_claim_name", + "jwt_claim_value", + "key" + ], + "title": "CreateJWTKeyMappingRequest", + "type": "object" + }, + "DeleteJWTKeyMappingRequest": { + "properties": { + "id": { + "title": "Id", + "type": "string" + } + }, + "required": [ + "id" + ], + "title": "DeleteJWTKeyMappingRequest", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "JWTKeyMappingResponse": { + "properties": { + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "title": "Id", + "type": "string" + }, + "is_active": { + "title": "Is Active", + "type": "boolean" + }, + "jwt_claim_name": { + "title": "Jwt Claim Name", + "type": "string" + }, + "jwt_claim_value": { + "title": "Jwt Claim Value", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "id", + "jwt_claim_name", + "jwt_claim_value", + "is_active", + "created_at", + "updated_at" + ], + "title": "JWTKeyMappingResponse", + "type": "object" + }, + "UpdateJWTKeyMappingRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "title": "Id", + "type": "string" + }, + "is_active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Is Active" + }, + "key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + } + }, + "required": [ + "id" + ], + "title": "UpdateJWTKeyMappingRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/jwt/key/mapping/delete": { + "post": { + "operationId": "delete_jwt_key_mapping_jwt_key_mapping_delete_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/info": { + "get": { + "operationId": "info_jwt_key_mapping_jwt_key_mapping_info_get", + "parameters": [ + { + "in": "query", + "name": "id", + "required": true, + "schema": { + "title": "Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Info Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/list": { + "get": { + "operationId": "list_jwt_key_mappings_jwt_key_mapping_list_get", + "parameters": [ + { + "description": "Page number", + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "description": "Page number", + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "description": "Page size", + "in": "query", + "name": "size", + "required": false, + "schema": { + "default": 50, + "description": "Page size", + "maximum": 100, + "minimum": 1, + "title": "Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Jwt Key Mappings", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/new": { + "post": { + "operationId": "create_jwt_key_mapping_jwt_key_mapping_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/update": { + "post": { + "operationId": "update_jwt_key_mapping_jwt_key_mapping_update_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + } + } + }, + "langfuse_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/langfuse/{endpoint}": { + "delete": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "get": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "patch": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "post": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "put": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + } + } + } + }, + "mcp_app": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/mcp-rest/test/connection": { + "post": { + "description": "Test if we can connect to the provided MCP server before adding it", + "operationId": "test_connection_mcp_rest_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Connection", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/test/tools/list": { + "post": { + "description": "Preview tools available from MCP server before adding it", + "operationId": "test_tools_list_mcp_rest_test_tools_list_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Tools List", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/tools/call": { + "post": { + "description": "REST API to call a specific MCP tool with the provided arguments", + "operationId": "call_tool_rest_api_mcp_rest_tools_call_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Call Tool Rest Api", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/tools/list": { + "get": { + "description": "List all available tools with information about the server they belong to.\n\nExample response:\n{\n \"tools\": [\n {\n \"name\": \"create_zap\",\n \"description\": \"Create a new zap\",\n \"inputSchema\": \"tool_input_schema\",\n \"mcp_info\": {\n \"server_name\": \"zapier\",\n \"logo_url\": \"https://www.zapier.com/logo.png\",\n }\n }\n ],\n \"error\": null,\n \"message\": \"Successfully retrieved tools\"\n}", + "operationId": "list_tool_rest_api_mcp_rest_tools_list_get", + "parameters": [ + { + "description": "The server id to list tools for", + "in": "query", + "name": "server_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The server id to list tools for", + "title": "Server Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response List Tool Rest Api Mcp Rest Tools List Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tool Rest Api", + "tags": [ + "mcp_app" + ] + } + } + } + }, + "mcp_byok_oauth": { + "components": { + "schemas": {} + }, + "paths": {} + }, + "mcp_discoverable": { + "components": { + "schemas": { + "CallbacksByType": { + "properties": { + "failure": { + "items": { + "type": "string" + }, + "title": "Failure", + "type": "array" + }, + "success": { + "items": { + "type": "string" + }, + "title": "Success", + "type": "array" + }, + "success_and_failure": { + "items": { + "type": "string" + }, + "title": "Success And Failure", + "type": "array" + } + }, + "required": [ + "success", + "failure", + "success_and_failure" + ], + "title": "CallbacksByType", + "type": "object" + } + } + }, + "paths": { + "/callbacks/configs": { + "get": { + "description": "Get Available Callback Configurations\n\nReturns the configuration details for all available logging callbacks,\nincluding supported parameters, field types, and descriptions.", + "operationId": "get_callback_configs_callbacks_configs_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Callback Configs", + "tags": [ + "mcp_discoverable" + ] + } + }, + "/callbacks/list": { + "get": { + "description": "View List of Active Logging Callbacks", + "operationId": "list_callbacks_callbacks_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CallbacksByType" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Callbacks", + "tags": [ + "mcp_discoverable" + ] + } + } + } + }, + "mcp_management": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_MCPServerTable": { + "description": "Represents a LiteLLM_MCPServerTable record", + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "items": { + "type": "string" + }, + "title": "Allowed Tools", + "type": "array" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "active", + "description": "Approval status: 'pending_review', 'active', 'rejected'", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "items": { + "type": "string" + }, + "title": "Extra Headers", + "type": "array" + }, + "has_user_credential": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Has User Credential" + }, + "health_check_error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Health Check Error" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "last_health_check": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Health Check" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "review_notes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Review Notes" + }, + "reviewed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed At" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "status": { + "anyOf": [ + { + "enum": [ + "healthy", + "unhealthy", + "unknown" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": "unknown", + "description": "Health status: 'healthy', 'unhealthy', 'unknown'", + "title": "Status" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By" + }, + "teams": { + "items": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + "title": "Teams", + "type": "array" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "required": [ + "server_id", + "transport" + ], + "title": "LiteLLM_MCPServerTable", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "MCPOAuthUserCredentialRequest": { + "description": "Stores a user's OAuth2 token for an OpenAPI MCP server.", + "properties": { + "access_token": { + "title": "Access Token", + "type": "string" + }, + "expires_in": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expires In" + }, + "refresh_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Refresh Token" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "required": [ + "access_token" + ], + "title": "MCPOAuthUserCredentialRequest", + "type": "object" + }, + "MCPOAuthUserCredentialStatus": { + "description": "Describes whether the calling user has a stored OAuth credential.", + "properties": { + "connected_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connected At" + }, + "expires_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "is_expired": { + "default": false, + "title": "Is Expired", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + } + }, + "required": [ + "server_id", + "has_credential" + ], + "title": "MCPOAuthUserCredentialStatus", + "type": "object" + }, + "MCPSubmissionsSummary": { + "properties": { + "active": { + "title": "Active", + "type": "integer" + }, + "items": { + "items": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + }, + "title": "Items", + "type": "array" + }, + "pending_review": { + "title": "Pending Review", + "type": "integer" + }, + "rejected": { + "title": "Rejected", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total", + "pending_review", + "active", + "rejected", + "items" + ], + "title": "MCPSubmissionsSummary", + "type": "object" + }, + "MCPToolsetTool": { + "properties": { + "server_id": { + "title": "Server Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "server_id", + "tool_name" + ], + "title": "MCPToolsetTool", + "type": "object" + }, + "MCPUserCredentialListItem": { + "description": "One entry in the /user-credentials list.", + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "connected_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connected At" + }, + "credential_type": { + "title": "Credential Type", + "type": "string" + }, + "expires_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + } + }, + "required": [ + "server_id", + "credential_type", + "has_credential" + ], + "title": "MCPUserCredentialListItem", + "type": "object" + }, + "MCPUserCredentialRequest": { + "properties": { + "credential": { + "title": "Credential", + "type": "string" + }, + "save": { + "default": true, + "title": "Save", + "type": "boolean" + } + }, + "required": [ + "credential" + ], + "title": "MCPUserCredentialRequest", + "type": "object" + }, + "MCPUserCredentialResponse": { + "properties": { + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + } + }, + "required": [ + "server_id", + "has_credential" + ], + "title": "MCPUserCredentialResponse", + "type": "object" + }, + "MakeMCPServersPublicRequest": { + "properties": { + "mcp_server_ids": { + "items": { + "type": "string" + }, + "title": "Mcp Server Ids", + "type": "array" + } + }, + "required": [ + "mcp_server_ids" + ], + "title": "MakeMCPServersPublicRequest", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "NewMCPToolsetRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "tools": { + "default": [], + "items": { + "$ref": "#/components/schemas/MCPToolsetTool" + }, + "title": "Tools", + "type": "array" + }, + "toolset_name": { + "title": "Toolset Name", + "type": "string" + } + }, + "required": [ + "toolset_name" + ], + "title": "NewMCPToolsetRequest", + "type": "object" + }, + "RejectMCPServerRequest": { + "properties": { + "review_notes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Review Notes" + } + }, + "title": "RejectMCPServerRequest", + "type": "object" + }, + "UpdateMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "required": [ + "server_id" + ], + "title": "UpdateMCPServerRequest", + "type": "object" + }, + "UpdateMCPToolsetRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "tools": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/MCPToolsetTool" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "toolset_id": { + "title": "Toolset Id", + "type": "string" + }, + "toolset_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Toolset Name" + } + }, + "required": [ + "toolset_id" + ], + "title": "UpdateMCPToolsetRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/mcp/access_groups": { + "get": { + "description": "Get all available MCP access groups from the database AND config", + "operationId": "get_mcp_access_groups_v1_mcp_access_groups_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Access Groups", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/discover": { + "get": { + "description": "Returns a curated list of well-known MCP servers for discovery UI", + "operationId": "discover_mcp_servers_v1_mcp_discover_get", + "parameters": [ + { + "description": "Search filter for server names and descriptions", + "in": "query", + "name": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Search filter for server names and descriptions", + "title": "Query" + } + }, + { + "description": "Filter by category", + "in": "query", + "name": "category", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Filter by category", + "title": "Category" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Discover Mcp Servers", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/make_public": { + "post": { + "description": "Allows making MCP servers public for AI Hub", + "operationId": "make_mcp_servers_public_v1_mcp_make_public_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MakeMCPServersPublicRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Mcp Servers Public", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/network/client-ip": { + "get": { + "description": "Returns the caller's IP address as seen by the proxy.", + "operationId": "get_client_ip_v1_mcp_network_client_ip_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Client Ip", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/openapi-registry": { + "get": { + "description": "Returns well-known OpenAPI APIs with OAuth 2.0 metadata for the OpenAPI MCP picker", + "operationId": "get_openapi_registry_v1_mcp_openapi_registry_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Openapi Registry", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/registry.json": { + "get": { + "description": "MCP registry endpoint. Spec: https://github.com/modelcontextprotocol/registry", + "operationId": "get_mcp_registry_v1_mcp_registry_json_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Mcp Registry", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server": { + "get": { + "description": "Returns the mcp server list with associated teams", + "operationId": "fetch_all_mcp_servers_v1_mcp_server_get", + "parameters": [ + { + "description": "Filter MCP servers by team scope. When provided, returns only servers the team has access to plus globally available (allow_all_keys) servers. Used by the Create Key UI to show team-scoped MCP servers.", + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Filter MCP servers by team scope. When provided, returns only servers the team has access to plus globally available (allow_all_keys) servers. Used by the Create Key UI to show team-scoped MCP servers.", + "title": "Team Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + }, + "title": "Response Fetch All Mcp Servers V1 Mcp Server Get", + "type": "array" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch All Mcp Servers", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Allows creation of mcp servers", + "operationId": "add_mcp_server_v1_mcp_server_post", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Mcp Server", + "tags": [ + "mcp_management" + ] + }, + "put": { + "description": "Allows deleting mcp serves in the db", + "operationId": "edit_mcp_server_v1_mcp_server_put", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/health": { + "get": { + "description": "Health check for MCP servers", + "operationId": "health_check_servers_v1_mcp_server_health_get", + "parameters": [ + { + "description": "Server IDs to check. If not provided, checks all accessible servers.", + "in": "query", + "name": "server_ids", + "required": false, + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Server IDs to check. If not provided, checks all accessible servers.", + "title": "Server Ids" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Health Check Servers", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/oauth/session": { + "post": { + "description": "Temporarily cache an MCP server in memory without writing to the database", + "operationId": "add_session_mcp_server_v1_mcp_server_oauth_session_post", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Session Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/register": { + "post": { + "description": "Submit a new MCP server for admin review (non-admin users). Mirrors POST /guardrails/register.", + "operationId": "register_mcp_server_v1_mcp_server_register_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/submissions": { + "get": { + "description": "Returns all MCP servers submitted by non-admin users (admin review queue). Mirrors GET /guardrails/submissions.", + "operationId": "get_mcp_server_submissions_v1_mcp_server_submissions_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPSubmissionsSummary" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Server Submissions", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}": { + "delete": { + "description": "Allows deleting mcp serves in the db", + "operationId": "remove_mcp_server_v1_mcp_server__server_id__delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + }, + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Remove Mcp Server", + "tags": [ + "mcp_management" + ] + }, + "get": { + "description": "Returns the mcp server info", + "operationId": "fetch_mcp_server_v1_mcp_server__server_id__get", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/approve": { + "put": { + "description": "Approve a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/approve.", + "operationId": "approve_mcp_server_submission_v1_mcp_server__server_id__approve_put", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Approve Mcp Server Submission", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/oauth-user-credential": { + "delete": { + "description": "Revoke the calling user's stored OAuth2 token for an MCP server", + "operationId": "delete_mcp_oauth_user_credential_v1_mcp_server__server_id__oauth_user_credential_delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Mcp Oauth User Credential", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Store the calling user's OAuth2 token for an OpenAPI MCP server", + "operationId": "store_mcp_oauth_user_credential_v1_mcp_server__server_id__oauth_user_credential_post", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Store Mcp Oauth User Credential", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/oauth-user-credential/status": { + "get": { + "description": "Check whether the calling user has a stored OAuth2 credential for this MCP server", + "operationId": "get_mcp_oauth_user_credential_status_v1_mcp_server__server_id__oauth_user_credential_status_get", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Oauth User Credential Status", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/reject": { + "put": { + "description": "Reject a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/reject.", + "operationId": "reject_mcp_server_submission_v1_mcp_server__server_id__reject_put", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejectMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Reject Mcp Server Submission", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/user-credential": { + "delete": { + "description": "Delete the calling user's stored API key for a BYOK MCP server", + "operationId": "delete_mcp_user_credential_v1_mcp_server__server_id__user_credential_delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Mcp User Credential", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Store or update the calling user's API key for a BYOK MCP server", + "operationId": "store_mcp_user_credential_v1_mcp_server__server_id__user_credential_post", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Store Mcp User Credential", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/tools": { + "get": { + "description": "Get all MCP tools available for the current key, including those from access groups", + "operationId": "get_mcp_tools_v1_mcp_tools_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Tools", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/toolset": { + "get": { + "description": "List MCP toolsets accessible to the calling key", + "operationId": "fetch_mcp_toolsets_v1_mcp_toolset_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Toolsets", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Create a new MCP toolset (admin only)", + "operationId": "add_mcp_toolset_v1_mcp_toolset_post", + "parameters": [ + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPToolsetRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Mcp Toolset", + "tags": [ + "mcp_management" + ] + }, + "put": { + "description": "Update an existing MCP toolset (admin only)", + "operationId": "edit_mcp_toolset_v1_mcp_toolset_put", + "parameters": [ + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMCPToolsetRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Mcp Toolset", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/toolset/{toolset_id}": { + "delete": { + "description": "Delete an MCP toolset (admin only)", + "operationId": "remove_mcp_toolset_v1_mcp_toolset__toolset_id__delete", + "parameters": [ + { + "in": "path", + "name": "toolset_id", + "required": true, + "schema": { + "title": "Toolset Id", + "type": "string" + } + }, + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Remove Mcp Toolset", + "tags": [ + "mcp_management" + ] + }, + "get": { + "description": "Get a specific MCP toolset by ID", + "operationId": "fetch_mcp_toolset_v1_mcp_toolset__toolset_id__get", + "parameters": [ + { + "in": "path", + "name": "toolset_id", + "required": true, + "schema": { + "title": "Toolset Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Toolset", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/user-credentials": { + "get": { + "description": "List all OAuth2 MCP credentials stored for the calling user", + "operationId": "list_mcp_user_credentials_v1_mcp_user_credentials_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/MCPUserCredentialListItem" + }, + "title": "Response List Mcp User Credentials V1 Mcp User Credentials Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Mcp User Credentials", + "tags": [ + "mcp_management" + ] + } + } + } + }, + "mcp_rest": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/mcp-rest/test/connection": { + "post": { + "description": "Test if we can connect to the provided MCP server before adding it", + "operationId": "test_connection_mcp_rest_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Connection", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/test/tools/list": { + "post": { + "description": "Preview tools available from MCP server before adding it", + "operationId": "test_tools_list_mcp_rest_test_tools_list_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Tools List", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/tools/call": { + "post": { + "description": "REST API to call a specific MCP tool with the provided arguments", + "operationId": "call_tool_rest_api_mcp_rest_tools_call_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Call Tool Rest Api", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/tools/list": { + "get": { + "description": "List all available tools with information about the server they belong to.\n\nExample response:\n{\n \"tools\": [\n {\n \"name\": \"create_zap\",\n \"description\": \"Create a new zap\",\n \"inputSchema\": \"tool_input_schema\",\n \"mcp_info\": {\n \"server_name\": \"zapier\",\n \"logo_url\": \"https://www.zapier.com/logo.png\",\n }\n }\n ],\n \"error\": null,\n \"message\": \"Successfully retrieved tools\"\n}", + "operationId": "list_tool_rest_api_mcp_rest_tools_list_get", + "parameters": [ + { + "description": "The server id to list tools for", + "in": "query", + "name": "server_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The server id to list tools for", + "title": "Server Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response List Tool Rest Api Mcp Rest Tools List Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tool Rest Api", + "tags": [ + "mcp_rest" + ] + } + } + } + }, + "policies": { + "components": { + "schemas": { + "ChatCompletionAssistantMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionRedactedThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageObject" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "function_call": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + { + "type": "null" + } + ] + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "reasoning_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reasoning Content" + }, + "reasoning_items": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionReasoningItem" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Reasoning Items" + }, + "role": { + "const": "assistant", + "title": "Role", + "type": "string" + }, + "thinking_blocks": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionRedactedThinkingBlock" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Thinking Blocks" + }, + "tool_calls": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionAssistantToolCall" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tool Calls" + } + }, + "required": [ + "role" + ], + "title": "ChatCompletionAssistantMessage", + "type": "object" + }, + "ChatCompletionAssistantToolCall": { + "properties": { + "function": { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "type": { + "const": "function", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "type", + "function" + ], + "title": "ChatCompletionAssistantToolCall", + "type": "object" + }, + "ChatCompletionAudioObject": { + "properties": { + "input_audio": { + "$ref": "#/components/schemas/InputAudio" + }, + "type": { + "const": "input_audio", + "title": "Type", + "type": "string" + } + }, + "required": [ + "input_audio", + "type" + ], + "title": "ChatCompletionAudioObject", + "type": "object" + }, + "ChatCompletionCachedContent": { + "properties": { + "type": { + "const": "ephemeral", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionCachedContent", + "type": "object" + }, + "ChatCompletionDeveloperMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": {}, + "type": "array" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "developer", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionDeveloperMessage", + "type": "object" + }, + "ChatCompletionDocumentObject": { + "properties": { + "citations": { + "anyOf": [ + { + "$ref": "#/components/schemas/CitationsObject" + }, + { + "type": "null" + } + ] + }, + "context": { + "title": "Context", + "type": "string" + }, + "source": { + "$ref": "#/components/schemas/DocumentObject" + }, + "title": { + "title": "Title", + "type": "string" + }, + "type": { + "const": "document", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "source", + "title", + "context", + "citations" + ], + "title": "ChatCompletionDocumentObject", + "type": "object" + }, + "ChatCompletionFileObject": { + "properties": { + "file": { + "$ref": "#/components/schemas/ChatCompletionFileObjectFile" + }, + "type": { + "const": "file", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "file" + ], + "title": "ChatCompletionFileObject", + "type": "object" + }, + "ChatCompletionFileObjectFile": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "file_data": { + "title": "File Data", + "type": "string" + }, + "file_id": { + "title": "File Id", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "format": { + "title": "Format", + "type": "string" + }, + "video_metadata": { + "additionalProperties": true, + "title": "Video Metadata", + "type": "object" + } + }, + "title": "ChatCompletionFileObjectFile", + "type": "object" + }, + "ChatCompletionFunctionMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "function", + "title": "Role", + "type": "string" + }, + "tool_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tool Call Id" + } + }, + "required": [ + "role", + "content", + "name", + "tool_call_id" + ], + "title": "ChatCompletionFunctionMessage", + "type": "object" + }, + "ChatCompletionImageObject": { + "properties": { + "image_url": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageUrlObject" + } + ], + "title": "Image Url" + }, + "type": { + "const": "image_url", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "ChatCompletionImageObject", + "type": "object" + }, + "ChatCompletionImageUrlObject": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "format": { + "title": "Format", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ChatCompletionImageUrlObject", + "type": "object" + }, + "ChatCompletionMessageToolCall": { + "additionalProperties": true, + "properties": {}, + "title": "ChatCompletionMessageToolCall", + "type": "object" + }, + "ChatCompletionReasoningItem": { + "description": "Represents an OpenAI Responses API reasoning item for round-tripping in conversation history.", + "properties": { + "encrypted_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Encrypted Content" + }, + "id": { + "title": "Id", + "type": "string" + }, + "summary": { + "items": { + "$ref": "#/components/schemas/ChatCompletionReasoningSummaryTextBlock" + }, + "title": "Summary", + "type": "array" + }, + "type": { + "const": "reasoning", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionReasoningItem", + "type": "object" + }, + "ChatCompletionReasoningSummaryTextBlock": { + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "summary_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionReasoningSummaryTextBlock", + "type": "object" + }, + "ChatCompletionRedactedThinkingBlock": { + "properties": { + "cache_control": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + { + "type": "null" + } + ], + "title": "Cache Control" + }, + "data": { + "title": "Data", + "type": "string" + }, + "type": { + "const": "redacted_thinking", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionRedactedThinkingBlock", + "type": "object" + }, + "ChatCompletionSystemMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": {}, + "type": "array" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "system", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionSystemMessage", + "type": "object" + }, + "ChatCompletionTextObject": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "ChatCompletionTextObject", + "type": "object" + }, + "ChatCompletionThinkingBlock": { + "properties": { + "cache_control": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + { + "type": "null" + } + ], + "title": "Cache Control" + }, + "signature": { + "title": "Signature", + "type": "string" + }, + "thinking": { + "title": "Thinking", + "type": "string" + }, + "type": { + "const": "thinking", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionThinkingBlock", + "type": "object" + }, + "ChatCompletionToolCallChunk": { + "properties": { + "function": { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "index": { + "title": "Index", + "type": "integer" + }, + "type": { + "const": "function", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "type", + "function", + "index" + ], + "title": "ChatCompletionToolCallChunk", + "type": "object" + }, + "ChatCompletionToolCallFunctionChunk": { + "properties": { + "arguments": { + "title": "Arguments", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "provider_specific_fields": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Provider Specific Fields" + } + }, + "title": "ChatCompletionToolCallFunctionChunk", + "type": "object" + }, + "ChatCompletionToolMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "tool", + "title": "Role", + "type": "string" + }, + "tool_call_id": { + "title": "Tool Call Id", + "type": "string" + } + }, + "required": [ + "role", + "content", + "tool_call_id" + ], + "title": "ChatCompletionToolMessage", + "type": "object" + }, + "ChatCompletionToolParam": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "function": { + "$ref": "#/components/schemas/ChatCompletionToolParamFunctionChunk" + }, + "type": { + "anyOf": [ + { + "const": "function", + "type": "string" + }, + { + "type": "string" + } + ], + "title": "Type" + } + }, + "required": [ + "type", + "function" + ], + "title": "ChatCompletionToolParam", + "type": "object" + }, + "ChatCompletionToolParamFunctionChunk": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "strict": { + "title": "Strict", + "type": "boolean" + } + }, + "required": [ + "name" + ], + "title": "ChatCompletionToolParamFunctionChunk", + "type": "object" + }, + "ChatCompletionUserMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionAudioObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionDocumentObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionVideoObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionFileObject" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionUserMessage", + "type": "object" + }, + "ChatCompletionVideoObject": { + "properties": { + "type": { + "const": "video_url", + "title": "Type", + "type": "string" + }, + "video_url": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/ChatCompletionVideoUrlObject" + } + ], + "title": "Video Url" + } + }, + "required": [ + "type", + "video_url" + ], + "title": "ChatCompletionVideoObject", + "type": "object" + }, + "ChatCompletionVideoUrlObject": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ChatCompletionVideoUrlObject", + "type": "object" + }, + "CitationsObject": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "title": "CitationsObject", + "type": "object" + }, + "DocumentObject": { + "properties": { + "data": { + "title": "Data", + "type": "string" + }, + "media_type": { + "title": "Media Type", + "type": "string" + }, + "type": { + "const": "text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "media_type", + "data" + ], + "title": "DocumentObject", + "type": "object" + }, + "EnrichTemplateRequest": { + "properties": { + "competitors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "maxItems": 100, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Optional list of competitor names", + "title": "Competitors" + }, + "instruction": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Refinement instruction for modifying the competitor list (e.g. 'add 10 more from Asia')", + "title": "Instruction" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "template_id": { + "title": "Template Id", + "type": "string" + } + }, + "required": [ + "template_id", + "parameters" + ], + "title": "EnrichTemplateRequest", + "type": "object" + }, + "GenericGuardrailAPIInputs": { + "properties": { + "images": { + "items": { + "type": "string" + }, + "title": "Images", + "type": "array" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "structured_messages": { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionUserMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionAssistantMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionToolMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionSystemMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionFunctionMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionDeveloperMessage" + } + ] + }, + "title": "Structured Messages", + "type": "array" + }, + "texts": { + "items": { + "type": "string" + }, + "title": "Texts", + "type": "array" + }, + "tool_calls": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionToolCallChunk" + }, + "type": "array" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionMessageToolCall" + }, + "type": "array" + } + ], + "title": "Tool Calls" + }, + "tools": { + "items": { + "$ref": "#/components/schemas/ChatCompletionToolParam" + }, + "title": "Tools", + "type": "array" + } + }, + "title": "GenericGuardrailAPIInputs", + "type": "object" + }, + "GuardrailTestResultEntry": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "details": { + "title": "Details", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "output_text": { + "title": "Output Text", + "type": "string" + } + }, + "required": [ + "guardrail_name", + "action", + "output_text", + "details" + ], + "title": "GuardrailTestResultEntry", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "InputAudio": { + "properties": { + "data": { + "title": "Data", + "type": "string" + }, + "format": { + "enum": [ + "wav", + "mp3" + ], + "title": "Format", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "InputAudio", + "type": "object" + }, + "PolicyGuardrailsResponse": { + "description": "Guardrails configuration for a policy.", + "properties": { + "add": { + "items": { + "type": "string" + }, + "title": "Add", + "type": "array" + }, + "remove": { + "items": { + "type": "string" + }, + "title": "Remove", + "type": "array" + } + }, + "title": "PolicyGuardrailsResponse", + "type": "object" + }, + "PolicyInfoResponse": { + "description": "Response for /policy/info/{policy_name} endpoint.", + "properties": { + "guardrails": { + "$ref": "#/components/schemas/PolicyGuardrailsResponse" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Inherit" + }, + "inheritance_chain": { + "items": { + "type": "string" + }, + "title": "Inheritance Chain", + "type": "array" + }, + "policy_name": { + "title": "Policy Name", + "type": "string" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + }, + "scope": { + "$ref": "#/components/schemas/PolicyScopeResponse" + } + }, + "required": [ + "policy_name", + "scope", + "guardrails", + "resolved_guardrails", + "inheritance_chain" + ], + "title": "PolicyInfoResponse", + "type": "object" + }, + "PolicyListResponse": { + "description": "Response for /policy/list endpoint.", + "properties": { + "policies": { + "additionalProperties": { + "$ref": "#/components/schemas/PolicySummaryItem" + }, + "title": "Policies", + "type": "object" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "policies", + "total_count" + ], + "title": "PolicyListResponse", + "type": "object" + }, + "PolicyMatchContext": { + "additionalProperties": false, + "description": "Context used to match a request against policies.\n\nContains the team alias, key alias, and model from the incoming request.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key alias from the request.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name from the request.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags from key/team metadata.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias from the request.", + "title": "Team Alias" + } + }, + "title": "PolicyMatchContext", + "type": "object" + }, + "PolicyScopeResponse": { + "description": "Scope configuration for a policy.", + "properties": { + "keys": { + "items": { + "type": "string" + }, + "title": "Keys", + "type": "array" + }, + "models": { + "items": { + "type": "string" + }, + "title": "Models", + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "teams": { + "items": { + "type": "string" + }, + "title": "Teams", + "type": "array" + } + }, + "title": "PolicyScopeResponse", + "type": "object" + }, + "PolicySummaryItem": { + "description": "Summary of a single policy for list endpoint.", + "properties": { + "guardrails": { + "$ref": "#/components/schemas/PolicyGuardrailsResponse" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Inherit" + }, + "inheritance_chain": { + "items": { + "type": "string" + }, + "title": "Inheritance Chain", + "type": "array" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + }, + "scope": { + "$ref": "#/components/schemas/PolicyScopeResponse" + } + }, + "required": [ + "scope", + "guardrails", + "resolved_guardrails", + "inheritance_chain" + ], + "title": "PolicySummaryItem", + "type": "object" + }, + "PolicyTestResponse": { + "description": "Response for /policy/test endpoint.", + "properties": { + "context": { + "$ref": "#/components/schemas/PolicyMatchContext" + }, + "matching_policies": { + "items": { + "type": "string" + }, + "title": "Matching Policies", + "type": "array" + }, + "message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Message" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + } + }, + "required": [ + "context", + "matching_policies", + "resolved_guardrails" + ], + "title": "PolicyTestResponse", + "type": "object" + }, + "PolicyValidateRequest": { + "additionalProperties": false, + "description": "Request body for the /policy/validate endpoint.", + "properties": { + "policies": { + "additionalProperties": true, + "description": "Policy configuration to validate. Map of policy names to policy definitions.", + "title": "Policies", + "type": "object" + } + }, + "required": [ + "policies" + ], + "title": "PolicyValidateRequest", + "type": "object" + }, + "PolicyValidationError": { + "additionalProperties": false, + "description": "Represents a validation error or warning for a policy.", + "properties": { + "error_type": { + "$ref": "#/components/schemas/PolicyValidationErrorType", + "description": "Type of validation error." + }, + "field": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Specific field that caused the error (e.g., 'guardrails.add', 'scope.teams').", + "title": "Field" + }, + "message": { + "description": "Human-readable error message.", + "title": "Message", + "type": "string" + }, + "policy_name": { + "description": "Name of the policy with the issue.", + "title": "Policy Name", + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The invalid value that caused the error.", + "title": "Value" + } + }, + "required": [ + "policy_name", + "error_type", + "message" + ], + "title": "PolicyValidationError", + "type": "object" + }, + "PolicyValidationErrorType": { + "description": "Types of validation errors that can occur.", + "enum": [ + "invalid_guardrail", + "invalid_team", + "invalid_key", + "invalid_model", + "invalid_inheritance", + "circular_inheritance", + "invalid_scope", + "invalid_syntax" + ], + "title": "PolicyValidationErrorType", + "type": "string" + }, + "PolicyValidationResponse": { + "additionalProperties": false, + "description": "Response from policy validation.\n\n- `valid`: True if no blocking errors were found\n- `errors`: List of blocking errors (prevent policy from being applied)\n- `warnings`: List of non-blocking warnings (policy can still be applied)", + "properties": { + "errors": { + "description": "List of blocking validation errors.", + "items": { + "$ref": "#/components/schemas/PolicyValidationError" + }, + "title": "Errors", + "type": "array" + }, + "valid": { + "description": "True if the policy configuration is valid.", + "title": "Valid", + "type": "boolean" + }, + "warnings": { + "description": "List of non-blocking validation warnings.", + "items": { + "$ref": "#/components/schemas/PolicyValidationError" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "valid" + ], + "title": "PolicyValidationResponse", + "type": "object" + }, + "SuggestTemplatesRequest": { + "properties": { + "attack_examples": { + "items": { + "type": "string" + }, + "title": "Attack Examples", + "type": "array" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + } + }, + "title": "SuggestTemplatesRequest", + "type": "object" + }, + "TestPoliciesAndGuardrailsRequest": { + "description": "Request body for POST /utils/test_policies_and_guardrails.", + "properties": { + "agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When set, call chat completion with this model/agent for each input and include the response in the result.", + "title": "Agent Id" + }, + "guardrail_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Guardrail names to apply directly", + "title": "Guardrail Names" + }, + "input_type": { + "default": "request", + "description": "Whether inputs are request or response", + "enum": [ + "request", + "response" + ], + "title": "Input Type", + "type": "string" + }, + "inputs_list": { + "default": [], + "description": "List of GenericGuardrailAPIInputs; each item processed separately (for batch compliance testing).", + "items": { + "$ref": "#/components/schemas/GenericGuardrailAPIInputs" + }, + "title": "Inputs List", + "type": "array" + }, + "policy_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Policy names to resolve guardrails from", + "title": "Policy Names" + }, + "request_data": { + "additionalProperties": true, + "description": "Request context (model, user_id, etc.)", + "title": "Request Data", + "type": "object" + } + }, + "title": "TestPoliciesAndGuardrailsRequest", + "type": "object" + }, + "TestPolicyTemplateRequest": { + "properties": { + "guardrail_definitions": { + "description": "All guardrailDefinitions from the policy template", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Guardrail Definitions", + "type": "array" + }, + "text": { + "description": "Test input text to run guardrails against", + "title": "Text", + "type": "string" + } + }, + "required": [ + "guardrail_definitions", + "text" + ], + "title": "TestPolicyTemplateRequest", + "type": "object" + }, + "TestPolicyTemplateResponse": { + "properties": { + "overall_action": { + "title": "Overall Action", + "type": "string" + }, + "results": { + "items": { + "$ref": "#/components/schemas/GuardrailTestResultEntry" + }, + "title": "Results", + "type": "array" + } + }, + "required": [ + "overall_action", + "results" + ], + "title": "TestPolicyTemplateResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policy/info/{policy_name}": { + "get": { + "description": "Get detailed information about a specific policy.\n\nReturns:\n- Policy configuration\n- Resolved guardrails (after inheritance)\n- Inheritance chain", + "operationId": "get_policy_info_policy_info__policy_name__get", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Info", + "tags": [ + "policies" + ] + } + }, + "/policy/list": { + "get": { + "description": "List all loaded policies with their resolved guardrails.\n\nReturns information about each policy including:\n- Inheritance configuration\n- Scope (teams, keys, models)\n- Guardrails to add/remove\n- Resolved guardrails (after inheritance)\n- Inheritance chain", + "operationId": "list_policies_policy_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyListResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policies", + "tags": [ + "policies" + ] + } + }, + "/policy/templates": { + "get": { + "description": "Get policy templates for the UI (pre-configured guardrail combinations).\n\nFetches from GitHub with automatic fallback to local backup on failure.\nSet LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.", + "operationId": "get_policy_templates_policy_templates_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": {}, + "title": "Response Get Policy Templates Policy Templates Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Templates", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/enrich": { + "post": { + "description": "Enrich a policy template with LLM-discovered data (e.g. competitor names).\n\nCalls an onboarded LLM to discover competitors for the given brand name,\nthen returns enriched guardrailDefinitions with the discovered data populated.", + "operationId": "enrich_policy_template_policy_templates_enrich_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnrichTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Enrich Policy Template Policy Templates Enrich Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enrich Policy Template", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/enrich/stream": { + "post": { + "description": "Stream competitor names as SSE events as the LLM generates them.\n\nEvents:\n- data: {\"type\": \"competitor\", \"name\": \"...\"} \u2014 each competitor as discovered\n- data: {\"type\": \"done\", \"competitors\": [...], \"competitor_variations\": {...}, \"guardrailDefinitions\": [...]}", + "operationId": "enrich_policy_template_stream_policy_templates_enrich_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnrichTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enrich Policy Template Stream", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/suggest": { + "post": { + "description": "Use AI to suggest policy templates based on attack examples and descriptions.\n\nCalls an LLM with tool calling to match user requirements to available templates.", + "operationId": "suggest_policy_templates_policy_templates_suggest_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuggestTemplatesRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Suggest Policy Templates Policy Templates Suggest Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Suggest Policy Templates", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/test": { + "post": { + "description": "Test a policy template's guardrails against a text input without creating them.\n\nInstantiates temporary guardrails from the template definitions, runs them\nagainst the provided text, and returns per-guardrail results so users can\nverify the template solves their problem before creating it.", + "operationId": "test_policy_template_policy_templates_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPolicyTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPolicyTemplateResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policy Template", + "tags": [ + "policies" + ] + } + }, + "/policy/test": { + "post": { + "description": "Test which policies would match a given request context.\n\nThis is useful for debugging and understanding policy behavior.\n\nRequest body:\n```json\n{\n \"team_alias\": \"healthcare-team\",\n \"key_alias\": \"my-api-key\",\n \"model\": \"gpt-4\"\n}\n```\n\nReturns:\n- matching_policies: List of policy names that match\n- resolved_guardrails: Final list of guardrails that would be applied", + "operationId": "test_policy_matching_policy_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyMatchContext" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyTestResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policy Matching", + "tags": [ + "policies" + ] + } + }, + "/policy/validate": { + "post": { + "description": "Validate a policy configuration before applying it.\n\nChecks:\n- All referenced guardrails exist in the guardrail registry\n- All non-wildcard team aliases exist in the database\n- All non-wildcard key aliases exist in the database\n- Inheritance chains are valid (no cycles, parents exist)\n- Scope patterns are syntactically valid\n\nReturns:\n- valid: True if the policy configuration is valid (no blocking errors)\n- errors: List of blocking validation errors\n- warnings: List of non-blocking validation warnings\n\nExample request:\n```json\n{\n \"policies\": {\n \"global-baseline\": {\n \"guardrails\": {\n \"add\": [\"pii_blocker\", \"phi_blocker\"]\n },\n \"scope\": {\n \"teams\": [\"*\"],\n \"keys\": [\"*\"],\n \"models\": [\"*\"]\n }\n },\n \"healthcare-compliance\": {\n \"inherit\": \"global-baseline\",\n \"guardrails\": {\n \"add\": [\"hipaa_audit\"]\n },\n \"scope\": {\n \"teams\": [\"healthcare-team\"]\n }\n }\n }\n}\n```", + "operationId": "validate_policy_policy_validate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyValidateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyValidationResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Validate Policy", + "tags": [ + "policies" + ] + } + }, + "/utils/test_policies_and_guardrails": { + "post": { + "description": "Apply policies and/or guardrails to inputs (for compliance UI testing).\n\nUse inputs_list for batch testing: each input is processed as a separate call so\nper-input block/allow and errors are returned.\n\nUse inputs for a single call (legacy).", + "operationId": "test_policies_and_guardrails_utils_test_policies_and_guardrails_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPoliciesAndGuardrailsRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policies And Guardrails", + "tags": [ + "policies" + ] + } + } + } + }, + "policy_engine": { + "components": { + "schemas": { + "AttachmentImpactResponse": { + "description": "Response for estimating the impact of a policy attachment.", + "properties": { + "affected_keys_count": { + "default": 0, + "description": "Number of keys that would be affected (named + unnamed).", + "title": "Affected Keys Count", + "type": "integer" + }, + "affected_teams_count": { + "default": 0, + "description": "Number of teams that would be affected (named + unnamed).", + "title": "Affected Teams Count", + "type": "integer" + }, + "sample_keys": { + "description": "Sample of affected key aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Keys", + "type": "array" + }, + "sample_teams": { + "description": "Sample of affected team aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Teams", + "type": "array" + }, + "unnamed_keys_count": { + "default": 0, + "description": "Number of affected keys without an alias.", + "title": "Unnamed Keys Count", + "type": "integer" + }, + "unnamed_teams_count": { + "default": 0, + "description": "Number of affected teams without an alias.", + "title": "Unnamed Teams Count", + "type": "integer" + } + }, + "title": "AttachmentImpactResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "PipelineTestRequest": { + "description": "Request body for testing a guardrail pipeline with sample messages.", + "properties": { + "pipeline": { + "additionalProperties": true, + "description": "Pipeline definition with 'mode' and 'steps'.", + "title": "Pipeline", + "type": "object" + }, + "test_messages": { + "description": "Test messages to run through the pipeline, e.g. [{'role': 'user', 'content': '...'}].", + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "title": "Test Messages", + "type": "array" + } + }, + "required": [ + "pipeline", + "test_messages" + ], + "title": "PipelineTestRequest", + "type": "object" + }, + "PolicyAttachmentCreateRequest": { + "description": "Request body for creating a policy attachment.", + "properties": { + "keys": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Key aliases or patterns this attachment applies to.", + "title": "Keys" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Model names or patterns this attachment applies to.", + "title": "Models" + }, + "policy_name": { + "description": "Name of the policy to attach.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Use '*' for global scope (applies to all requests).", + "title": "Scope" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tag patterns this attachment applies to. Supports wildcards (e.g., health-*).", + "title": "Tags" + }, + "teams": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Team aliases or patterns this attachment applies to.", + "title": "Teams" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyAttachmentCreateRequest", + "type": "object" + }, + "PolicyAttachmentDBResponse": { + "description": "Response for a policy attachment from the database.", + "properties": { + "attachment_id": { + "description": "Unique ID of the attachment.", + "title": "Attachment Id", + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the attachment was created.", + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who created the attachment.", + "title": "Created By" + }, + "keys": { + "description": "Key patterns.", + "items": { + "type": "string" + }, + "title": "Keys", + "type": "array" + }, + "models": { + "description": "Model patterns.", + "items": { + "type": "string" + }, + "title": "Models", + "type": "array" + }, + "policy_name": { + "description": "Name of the attached policy.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Scope of the attachment.", + "title": "Scope" + }, + "tags": { + "description": "Tag patterns.", + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "teams": { + "description": "Team patterns.", + "items": { + "type": "string" + }, + "title": "Teams", + "type": "array" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the attachment was last updated.", + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who last updated the attachment.", + "title": "Updated By" + } + }, + "required": [ + "attachment_id", + "policy_name" + ], + "title": "PolicyAttachmentDBResponse", + "type": "object" + }, + "PolicyAttachmentListResponse": { + "description": "Response for listing policy attachments.", + "properties": { + "attachments": { + "description": "List of policy attachments.", + "items": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + }, + "title": "Attachments", + "type": "array" + }, + "total_count": { + "default": 0, + "description": "Total number of attachments.", + "title": "Total Count", + "type": "integer" + } + }, + "title": "PolicyAttachmentListResponse", + "type": "object" + }, + "PolicyConditionRequest": { + "description": "Condition for when a policy applies.", + "properties": { + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name pattern (exact match or regex) for when policy applies.", + "title": "Model" + } + }, + "title": "PolicyConditionRequest", + "type": "object" + }, + "PolicyCreateRequest": { + "description": "Request body for creating a new policy.", + "properties": { + "condition": { + "anyOf": [ + { + "$ref": "#/components/schemas/PolicyConditionRequest" + }, + { + "type": "null" + } + ], + "description": "Condition for when this policy applies." + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Human-readable description of the policy.", + "title": "Description" + }, + "guardrails_add": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to add.", + "title": "Guardrails Add" + }, + "guardrails_remove": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to remove (from inherited).", + "title": "Guardrails Remove" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of parent policy to inherit from.", + "title": "Inherit" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline for ordered execution. Contains 'mode' and 'steps'.", + "title": "Pipeline" + }, + "policy_name": { + "description": "Unique name for the policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyCreateRequest", + "type": "object" + }, + "PolicyDBResponse": { + "description": "Response for a policy from the database.", + "properties": { + "condition": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Policy condition.", + "title": "Condition" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the policy was created.", + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who created the policy.", + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy description.", + "title": "Description" + }, + "guardrails_add": { + "description": "Guardrails to add.", + "items": { + "type": "string" + }, + "title": "Guardrails Add", + "type": "array" + }, + "guardrails_remove": { + "description": "Guardrails to remove.", + "items": { + "type": "string" + }, + "title": "Guardrails Remove", + "type": "array" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Parent policy name.", + "title": "Inherit" + }, + "is_latest": { + "default": true, + "description": "True if this is the latest version by version_number.", + "title": "Is Latest", + "type": "boolean" + }, + "parent_version_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy ID this version was cloned from.", + "title": "Parent Version Id" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline.", + "title": "Pipeline" + }, + "policy_id": { + "description": "Unique ID of the policy.", + "title": "Policy Id", + "type": "string" + }, + "policy_name": { + "description": "Name of the policy.", + "title": "Policy Name", + "type": "string" + }, + "production_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When this version was promoted to production.", + "title": "Production At" + }, + "published_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When this version was published.", + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the policy was last updated.", + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who last updated the policy.", + "title": "Updated By" + }, + "version_number": { + "default": 1, + "description": "Version number of this policy.", + "title": "Version Number", + "type": "integer" + }, + "version_status": { + "default": "production", + "description": "One of: draft, published, production.", + "title": "Version Status", + "type": "string" + } + }, + "required": [ + "policy_id", + "policy_name" + ], + "title": "PolicyDBResponse", + "type": "object" + }, + "PolicyListDBResponse": { + "description": "Response for listing policies from the database.", + "properties": { + "policies": { + "description": "List of policies.", + "items": { + "$ref": "#/components/schemas/PolicyDBResponse" + }, + "title": "Policies", + "type": "array" + }, + "total_count": { + "default": 0, + "description": "Total number of policies.", + "title": "Total Count", + "type": "integer" + } + }, + "title": "PolicyListDBResponse", + "type": "object" + }, + "PolicyMatchDetail": { + "description": "Details about why a specific policy matched.", + "properties": { + "guardrails_added": { + "description": "Guardrails this policy contributes.", + "items": { + "type": "string" + }, + "title": "Guardrails Added", + "type": "array" + }, + "matched_via": { + "description": "How the policy was matched (e.g., 'tag:healthcare', 'team:health-team', 'scope:*').", + "title": "Matched Via", + "type": "string" + }, + "policy_name": { + "description": "Name of the matched policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name", + "matched_via" + ], + "title": "PolicyMatchDetail", + "type": "object" + }, + "PolicyResolveRequest": { + "description": "Request body for resolving effective policies/guardrails for a context.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key alias to resolve for.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name to resolve for.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags to resolve for.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias to resolve for.", + "title": "Team Alias" + } + }, + "title": "PolicyResolveRequest", + "type": "object" + }, + "PolicyResolveResponse": { + "description": "Response for resolving effective policies/guardrails for a context.", + "properties": { + "effective_guardrails": { + "description": "Final list of guardrails that would be applied.", + "items": { + "type": "string" + }, + "title": "Effective Guardrails", + "type": "array" + }, + "matched_policies": { + "description": "Details about each matched policy and why it matched.", + "items": { + "$ref": "#/components/schemas/PolicyMatchDetail" + }, + "title": "Matched Policies", + "type": "array" + } + }, + "title": "PolicyResolveResponse", + "type": "object" + }, + "PolicyUpdateRequest": { + "description": "Request body for updating a policy.", + "properties": { + "condition": { + "anyOf": [ + { + "$ref": "#/components/schemas/PolicyConditionRequest" + }, + { + "type": "null" + } + ], + "description": "Condition for when this policy applies." + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Human-readable description of the policy.", + "title": "Description" + }, + "guardrails_add": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to add.", + "title": "Guardrails Add" + }, + "guardrails_remove": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to remove (from inherited).", + "title": "Guardrails Remove" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of parent policy to inherit from.", + "title": "Inherit" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline for ordered execution. Contains 'mode' and 'steps'.", + "title": "Pipeline" + }, + "policy_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New name for the policy.", + "title": "Policy Name" + } + }, + "title": "PolicyUpdateRequest", + "type": "object" + }, + "PolicyVersionCompareResponse": { + "description": "Response for comparing two policy versions.", + "properties": { + "field_diffs": { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "description": "Field name -> {version_a: val, version_b: val} for differing fields.", + "title": "Field Diffs", + "type": "object" + }, + "version_a": { + "$ref": "#/components/schemas/PolicyDBResponse", + "description": "First version." + }, + "version_b": { + "$ref": "#/components/schemas/PolicyDBResponse", + "description": "Second version." + } + }, + "required": [ + "version_a", + "version_b" + ], + "title": "PolicyVersionCompareResponse", + "type": "object" + }, + "PolicyVersionCreateRequest": { + "description": "Request body for creating a new policy version (draft).", + "properties": { + "source_policy_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy ID to clone from. If None, clone from current production version.", + "title": "Source Policy Id" + } + }, + "title": "PolicyVersionCreateRequest", + "type": "object" + }, + "PolicyVersionListResponse": { + "description": "Response for listing all versions of a policy.", + "properties": { + "policy_name": { + "description": "Name of the policy.", + "title": "Policy Name", + "type": "string" + }, + "total_count": { + "default": 0, + "description": "Total number of versions.", + "title": "Total Count", + "type": "integer" + }, + "versions": { + "description": "All versions ordered by version_number desc.", + "items": { + "$ref": "#/components/schemas/PolicyDBResponse" + }, + "title": "Versions", + "type": "array" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyVersionListResponse", + "type": "object" + }, + "PolicyVersionStatusUpdateRequest": { + "description": "Request body for updating a policy version's status.", + "properties": { + "version_status": { + "description": "New status: 'published' or 'production'.", + "title": "Version Status", + "type": "string" + } + }, + "required": [ + "version_status" + ], + "title": "PolicyVersionStatusUpdateRequest", + "type": "object" + }, + "UsageOverviewResponse": { + "properties": { + "chart": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Chart", + "type": "array" + }, + "passRate": { + "title": "Passrate", + "type": "number" + }, + "rows": { + "items": { + "$ref": "#/components/schemas/UsageOverviewRow" + }, + "title": "Rows", + "type": "array" + }, + "totalBlocked": { + "title": "Totalblocked", + "type": "integer" + }, + "totalRequests": { + "title": "Totalrequests", + "type": "integer" + } + }, + "required": [ + "rows", + "chart", + "totalRequests", + "totalBlocked", + "passRate" + ], + "title": "UsageOverviewResponse", + "type": "object" + }, + "UsageOverviewRow": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend" + ], + "title": "UsageOverviewRow", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policies": { + "post": { + "description": "Create a new policy.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"global-baseline\",\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\", \"prompt_injection\"],\n \"guardrails_remove\": []\n }'\n```\n\nExample Response:\n```json\n{\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"inherit\": null,\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\", \"prompt_injection\"],\n \"guardrails_remove\": [],\n \"condition\": null,\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n}\n```", + "operationId": "create_policy_policies_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments": { + "post": { + "description": "Create a new policy attachment.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\"\n }'\n```\n\nExample with team-specific attachment:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"healthcare-compliance\",\n \"teams\": [\"healthcare-team\", \"medical-research\"]\n }'\n```\n\nExample Response:\n```json\n{\n \"attachment_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\",\n \"teams\": [],\n \"keys\": [],\n \"models\": [],\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n}\n```", + "operationId": "create_policy_attachment_policies_attachments_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy Attachment", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/estimate-impact": { + "post": { + "description": "Estimate how many keys and teams would be affected by a policy attachment.\n\nUse this before creating an attachment to preview the blast radius.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments/estimate-impact\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"hipaa-compliance\",\n \"tags\": [\"healthcare\", \"health-*\"]\n }'\n```", + "operationId": "estimate_attachment_impact_policies_attachments_estimate_impact_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachmentImpactResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Estimate Attachment Impact", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/list": { + "get": { + "description": "List all policy attachments from the database.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/attachments/list\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"attachments\": [\n {\n \"attachment_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\",\n \"teams\": [],\n \"keys\": [],\n \"models\": [],\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n }\n ],\n \"total_count\": 1\n}\n```", + "operationId": "list_policy_attachments_policies_attachments_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentListResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policy Attachments", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/{attachment_id}": { + "delete": { + "description": "Delete a policy attachment.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/policies/attachments/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Attachment 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_policy_attachment_policies_attachments__attachment_id__delete", + "parameters": [ + { + "in": "path", + "name": "attachment_id", + "required": true, + "schema": { + "title": "Attachment Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Policy Attachment", + "tags": [ + "policy_engine" + ] + }, + "get": { + "description": "Get a policy attachment by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/attachments/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_policy_attachment_policies_attachments__attachment_id__get", + "parameters": [ + { + "in": "path", + "name": "attachment_id", + "required": true, + "schema": { + "title": "Attachment Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Attachment", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/compare": { + "get": { + "description": "Compare two policy versions. Query params: version_a, version_b (policy version IDs).", + "operationId": "compare_policy_versions_policies_compare_get", + "parameters": [ + { + "in": "query", + "name": "version_a", + "required": true, + "schema": { + "title": "Version A", + "type": "string" + } + }, + { + "in": "query", + "name": "version_b", + "required": true, + "schema": { + "title": "Version B", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionCompareResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Compare Policy Versions", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/list": { + "get": { + "description": "List all policies from the database. Optionally filter by version_status.\n\nQuery params:\n- version_status: Optional. One of \"draft\", \"published\", \"production\".\n If omitted, all versions are returned.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/list\" \\\n -H \"Authorization: Bearer \"\ncurl -X GET \"http://localhost:4000/policies/list?version_status=production\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"policies\": [\n {\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"version_number\": 1,\n \"version_status\": \"production\",\n \"inherit\": null,\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\"],\n \"guardrails_remove\": [],\n \"condition\": null,\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n }\n ],\n \"total_count\": 1\n}\n```", + "operationId": "list_policies_policies_list_get", + "parameters": [ + { + "in": "query", + "name": "version_status", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version Status" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyListDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policies", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/name/{policy_name}/all-versions": { + "delete": { + "description": "Delete all versions of a policy. Also removes from in-memory registry.", + "operationId": "delete_all_policy_versions_policies_name__policy_name__all_versions_delete", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete All Policy Versions", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/name/{policy_name}/versions": { + "get": { + "description": "List all versions of a policy by name, ordered by version_number descending.", + "operationId": "list_policy_versions_policies_name__policy_name__versions_get", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policy Versions", + "tags": [ + "policy_engine" + ] + }, + "post": { + "description": "Create a new draft version of a policy. Copies all fields from the source.\nSource is current production if source_policy_id is not provided.", + "operationId": "create_policy_version_policies_name__policy_name__versions_post", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy Version", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/resolve": { + "post": { + "description": "Resolve which policies and guardrails apply for a given context.\n\nUse this endpoint to debug \"what guardrails would apply to a request\nwith this team/key/model/tags combination?\"\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/resolve\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"tags\": [\"healthcare\"],\n \"model\": \"gpt-4\"\n }'\n```", + "operationId": "resolve_policies_for_context_policies_resolve_post", + "parameters": [ + { + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "in": "query", + "name": "force_sync", + "required": false, + "schema": { + "default": false, + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "title": "Force Sync", + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Resolve Policies For Context", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/test-pipeline": { + "post": { + "description": "Test a guardrail pipeline with sample messages.\n\nExecutes the pipeline steps against the provided test messages and returns\nstep-by-step results showing which guardrails passed/failed, actions taken,\nand timing information.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/test-pipeline\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"pipeline\": {\n \"mode\": \"pre_call\",\n \"steps\": [\n {\"guardrail\": \"pii-guard\", \"on_pass\": \"next\", \"on_fail\": \"block\"}\n ]\n },\n \"test_messages\": [{\"role\": \"user\", \"content\": \"My SSN is 123-45-6789\"}]\n }'\n```", + "operationId": "test_pipeline_policies_test_pipeline_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PipelineTestRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Pipeline", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/usage/overview": { + "get": { + "description": "Return policy performance overview for the dashboard.", + "operationId": "policies_usage_overview_policies_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Policies Usage Overview", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}": { + "delete": { + "description": "Delete a policy.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Policy 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_policy_policies__policy_id__delete", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Policy", + "tags": [ + "policy_engine" + ] + }, + "get": { + "description": "Get a policy by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_policy_policies__policy_id__get", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy", + "tags": [ + "policy_engine" + ] + }, + "put": { + "description": "Update an existing policy.\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"description\": \"Updated description\",\n \"guardrails_add\": [\"pii_masking\", \"toxicity_filter\"]\n }'\n```", + "operationId": "update_policy_policies__policy_id__put", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Policy", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}/resolved-guardrails": { + "get": { + "description": "Get the resolved guardrails for a policy (including inherited guardrails).\n\nThis endpoint resolves the full inheritance chain and returns the final\nset of guardrails that would be applied for this policy.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000/resolved-guardrails\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"healthcare-compliance\",\n \"resolved_guardrails\": [\"pii_masking\", \"prompt_injection\", \"toxicity_filter\"]\n}\n```", + "operationId": "get_resolved_guardrails_policies__policy_id__resolved_guardrails_get", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resolved Guardrails", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}/status": { + "put": { + "description": "Update a policy version's status. Valid transitions:\n- draft -> published\n- published -> production (demotes current production to published)\n- production -> published (demotes, policy becomes inactive)", + "operationId": "update_policy_version_status_policies__policy_id__status_put", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionStatusUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Policy Version Status", + "tags": [ + "policy_engine" + ] + } + } + } + }, + "policy_resolve": { + "components": { + "schemas": { + "AttachmentImpactResponse": { + "description": "Response for estimating the impact of a policy attachment.", + "properties": { + "affected_keys_count": { + "default": 0, + "description": "Number of keys that would be affected (named + unnamed).", + "title": "Affected Keys Count", + "type": "integer" + }, + "affected_teams_count": { + "default": 0, + "description": "Number of teams that would be affected (named + unnamed).", + "title": "Affected Teams Count", + "type": "integer" + }, + "sample_keys": { + "description": "Sample of affected key aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Keys", + "type": "array" + }, + "sample_teams": { + "description": "Sample of affected team aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Teams", + "type": "array" + }, + "unnamed_keys_count": { + "default": 0, + "description": "Number of affected keys without an alias.", + "title": "Unnamed Keys Count", + "type": "integer" + }, + "unnamed_teams_count": { + "default": 0, + "description": "Number of affected teams without an alias.", + "title": "Unnamed Teams Count", + "type": "integer" + } + }, + "title": "AttachmentImpactResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "PolicyAttachmentCreateRequest": { + "description": "Request body for creating a policy attachment.", + "properties": { + "keys": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Key aliases or patterns this attachment applies to.", + "title": "Keys" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Model names or patterns this attachment applies to.", + "title": "Models" + }, + "policy_name": { + "description": "Name of the policy to attach.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Use '*' for global scope (applies to all requests).", + "title": "Scope" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tag patterns this attachment applies to. Supports wildcards (e.g., health-*).", + "title": "Tags" + }, + "teams": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Team aliases or patterns this attachment applies to.", + "title": "Teams" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyAttachmentCreateRequest", + "type": "object" + }, + "PolicyMatchDetail": { + "description": "Details about why a specific policy matched.", + "properties": { + "guardrails_added": { + "description": "Guardrails this policy contributes.", + "items": { + "type": "string" + }, + "title": "Guardrails Added", + "type": "array" + }, + "matched_via": { + "description": "How the policy was matched (e.g., 'tag:healthcare', 'team:health-team', 'scope:*').", + "title": "Matched Via", + "type": "string" + }, + "policy_name": { + "description": "Name of the matched policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name", + "matched_via" + ], + "title": "PolicyMatchDetail", + "type": "object" + }, + "PolicyResolveRequest": { + "description": "Request body for resolving effective policies/guardrails for a context.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key alias to resolve for.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name to resolve for.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags to resolve for.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias to resolve for.", + "title": "Team Alias" + } + }, + "title": "PolicyResolveRequest", + "type": "object" + }, + "PolicyResolveResponse": { + "description": "Response for resolving effective policies/guardrails for a context.", + "properties": { + "effective_guardrails": { + "description": "Final list of guardrails that would be applied.", + "items": { + "type": "string" + }, + "title": "Effective Guardrails", + "type": "array" + }, + "matched_policies": { + "description": "Details about each matched policy and why it matched.", + "items": { + "$ref": "#/components/schemas/PolicyMatchDetail" + }, + "title": "Matched Policies", + "type": "array" + } + }, + "title": "PolicyResolveResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policies/attachments/estimate-impact": { + "post": { + "description": "Estimate how many keys and teams would be affected by a policy attachment.\n\nUse this before creating an attachment to preview the blast radius.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments/estimate-impact\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"hipaa-compliance\",\n \"tags\": [\"healthcare\", \"health-*\"]\n }'\n```", + "operationId": "estimate_attachment_impact_policies_attachments_estimate_impact_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachmentImpactResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Estimate Attachment Impact", + "tags": [ + "policy_resolve" + ] + } + }, + "/policies/resolve": { + "post": { + "description": "Resolve which policies and guardrails apply for a given context.\n\nUse this endpoint to debug \"what guardrails would apply to a request\nwith this team/key/model/tags combination?\"\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/resolve\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"tags\": [\"healthcare\"],\n \"model\": \"gpt-4\"\n }'\n```", + "operationId": "resolve_policies_for_context_policies_resolve_post", + "parameters": [ + { + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "in": "query", + "name": "force_sync", + "required": false, + "schema": { + "default": false, + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "title": "Force Sync", + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Resolve Policies For Context", + "tags": [ + "policy_resolve" + ] + } + } + } + }, + "prompts": { + "components": { + "schemas": { + "Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post": { + "properties": { + "file": { + "format": "binary", + "title": "File", + "type": "string" + } + }, + "required": [ + "file" + ], + "title": "Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListPromptsResponse": { + "properties": { + "prompts": { + "items": { + "$ref": "#/components/schemas/PromptSpec" + }, + "title": "Prompts", + "type": "array" + } + }, + "required": [ + "prompts" + ], + "title": "ListPromptsResponse", + "type": "object" + }, + "PatchPromptRequest": { + "properties": { + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + { + "type": "null" + } + ] + }, + "prompt_info": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptInfo" + }, + { + "type": "null" + } + ] + } + }, + "title": "PatchPromptRequest", + "type": "object" + }, + "Prompt": { + "properties": { + "litellm_params": { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + "prompt_id": { + "title": "Prompt Id", + "type": "string" + }, + "prompt_info": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptInfo" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "prompt_id", + "litellm_params" + ], + "title": "Prompt", + "type": "object" + }, + "PromptInfo": { + "additionalProperties": true, + "properties": { + "environment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "development", + "title": "Environment" + }, + "prompt_type": { + "enum": [ + "config", + "db" + ], + "title": "Prompt Type", + "type": "string" + } + }, + "required": [ + "prompt_type" + ], + "title": "PromptInfo", + "type": "object" + }, + "PromptInfoResponse": { + "properties": { + "environments": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Environments" + }, + "prompt_spec": { + "$ref": "#/components/schemas/PromptSpec" + }, + "raw_prompt_template": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptTemplateBase" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "prompt_spec" + ], + "title": "PromptInfoResponse", + "type": "object" + }, + "PromptLiteLLMParams": { + "additionalProperties": true, + "properties": { + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Base" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + }, + "dotprompt_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Dotprompt Content" + }, + "ignore_prompt_manager_model": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Ignore Prompt Manager Model" + }, + "ignore_prompt_manager_optional_params": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Ignore Prompt Manager Optional Params" + }, + "prompt_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt Id" + }, + "prompt_integration": { + "title": "Prompt Integration", + "type": "string" + }, + "provider_specific_query_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Provider Specific Query Params" + } + }, + "required": [ + "prompt_integration" + ], + "title": "PromptLiteLLMParams", + "type": "object" + }, + "PromptSpec": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "environment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "development", + "title": "Environment" + }, + "litellm_params": { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + "prompt_id": { + "title": "Prompt Id", + "type": "string" + }, + "prompt_info": { + "$ref": "#/components/schemas/PromptInfo" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "version": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "prompt_id", + "litellm_params", + "prompt_info" + ], + "title": "PromptSpec", + "type": "object" + }, + "PromptTemplateBase": { + "properties": { + "content": { + "title": "Content", + "type": "string" + }, + "litellm_prompt_id": { + "title": "Litellm Prompt Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + } + }, + "required": [ + "litellm_prompt_id", + "content" + ], + "title": "PromptTemplateBase", + "type": "object" + }, + "TestPromptRequest": { + "properties": { + "conversation_history": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Conversation History" + }, + "dotprompt_content": { + "title": "Dotprompt Content", + "type": "string" + }, + "prompt_variables": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt Variables" + } + }, + "required": [ + "dotprompt_content" + ], + "title": "TestPromptRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/prompts": { + "post": { + "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", + "operationId": "create_prompt_prompts_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Prompt" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/list": { + "get": { + "description": "List the prompts that are available on the proxy server\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/prompts/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"prompts\": [\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n }\n ]\n}\n```", + "operationId": "list_prompts_prompts_list_get", + "parameters": [ + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPromptsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Prompts", + "tags": [ + "prompts" + ] + } + }, + "/prompts/test": { + "post": { + "description": "Test a prompt by rendering it with variables and executing an LLM call.\n\nThis endpoint allows testing prompts before saving them to the database.\nThe response is always streamed.\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts/test\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"dotprompt_content\": \"---\\nmodel: gpt-4o\\ntemperature: 0.7\\n---\\n\\nUser: Hello {{name}}\",\n \"prompt_variables\": {\n \"name\": \"World\"\n }\n }'\n```", + "operationId": "test_prompt_prompts_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPromptRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}": { + "delete": { + "description": "Delete a prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Prompt my_prompt_id deleted successfully\"\n}\n```", + "operationId": "delete_prompt_prompts__prompt_id__delete", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Prompt", + "tags": [ + "prompts" + ] + }, + "get": { + "description": "Get detailed information about a specific prompt by ID, including prompt content\n\n \ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\n Example Request:\n ```bash\n curl -X GET \"http://localhost:4000/prompts/my_prompt_id/info\" \\\n -H \"Authorization: Bearer \"\n ```\n\n Example Response:\n ```json\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"content\": \"System: You are a helpful assistant.\n\nUser: {{user_message}}\"\n }\n ```", + "operationId": "get_prompt_info_prompts__prompt_id__get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PromptInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Info", + "tags": [ + "prompts" + ] + }, + "patch": { + "description": "Partially update an existing prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nThis endpoint allows updating specific fields of a prompt without sending the entire object.\nOnly the following fields can be updated:\n- litellm_params: LiteLLM parameters for the prompt\n- prompt_info: Additional information about the prompt\n\nExample Request:\n```bash\ncurl -X PATCH \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_info\": {\n \"prompt_type\": \"db\"\n }\n }'\n```", + "operationId": "patch_prompt_prompts__prompt_id__patch", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchPromptRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Prompt", + "tags": [ + "prompts" + ] + }, + "put": { + "description": "Update an existing prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }\n }'\n```", + "operationId": "update_prompt_prompts__prompt_id__put", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Prompt" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}/info": { + "get": { + "description": "Get detailed information about a specific prompt by ID, including prompt content\n\n \ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\n Example Request:\n ```bash\n curl -X GET \"http://localhost:4000/prompts/my_prompt_id/info\" \\\n -H \"Authorization: Bearer \"\n ```\n\n Example Response:\n ```json\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"content\": \"System: You are a helpful assistant.\n\nUser: {{user_message}}\"\n }\n ```", + "operationId": "get_prompt_info_prompts__prompt_id__info_get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PromptInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Info", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}/versions": { + "get": { + "description": "Get all versions of a specific prompt by base prompt ID\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/prompts/jack_success/versions\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"prompts\": [\n {\n \"prompt_id\": \"jack_success.v1\",\n \"litellm_params\": {...},\n \"prompt_info\": {\"prompt_type\": \"db\"},\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n },\n {\n \"prompt_id\": \"jack_success.v2\",\n \"litellm_params\": {...},\n \"prompt_info\": {\"prompt_type\": \"db\"},\n \"created_at\": \"2023-11-09T13:45:12.345Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n }\n ]\n}\n```", + "operationId": "get_prompt_versions_prompts__prompt_id__versions_get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPromptsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Versions", + "tags": [ + "prompts" + ] + } + }, + "/utils/dotprompt_json_converter": { + "post": { + "description": "Convert a .prompt file to JSON format.\n\nThis endpoint accepts a .prompt file upload and returns the equivalent JSON representation\nthat can be stored in a database or used programmatically.\n\nReturns the JSON structure with 'content' and 'metadata' fields.", + "operationId": "convert_prompt_file_to_json_utils_dotprompt_json_converter_post", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Convert Prompt File To Json Utils Dotprompt Json Converter Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Convert Prompt File To Json", + "tags": [ + "prompts" + ] + } + } + } + }, + "realtime": { + "components": { + "schemas": { + "RealtimeClientSecretResponse": { + "description": "Response from POST /v1/realtime/client_secrets.\n\nBoth the top-level `value` and `session.client_secret.value`\nwill contain the encrypted token instead of the raw ephemeral key.\nThe `session` field is kept as a raw dict so unknown fields pass through.", + "properties": { + "expires_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Session" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "RealtimeClientSecretResponse", + "type": "object" + } + } + }, + "paths": { + "/openai/v1/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/openai/v1/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, + "/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_v1_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_v1_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + } + } + }, + "scim": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "SCIMFeature": { + "properties": { + "maxOperations": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxoperations" + }, + "maxPayloadSize": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxpayloadsize" + }, + "maxResults": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxresults" + }, + "supported": { + "title": "Supported", + "type": "boolean" + } + }, + "required": [ + "supported" + ], + "title": "SCIMFeature", + "type": "object" + }, + "SCIMGroup": { + "properties": { + "displayName": { + "title": "Displayname", + "type": "string" + }, + "externalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Externalid" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "members": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMMember" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Members" + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "schemas": { + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + } + }, + "required": [ + "schemas", + "displayName" + ], + "title": "SCIMGroup", + "type": "object" + }, + "SCIMListResponse": { + "properties": { + "Resources": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUser" + }, + "type": "array" + }, + { + "items": { + "$ref": "#/components/schemas/SCIMGroup" + }, + "type": "array" + } + ], + "title": "Resources" + }, + "itemsPerPage": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10, + "title": "Itemsperpage" + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:api:messages:2.0:ListResponse" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "startIndex": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1, + "title": "Startindex" + }, + "totalResults": { + "title": "Totalresults", + "type": "integer" + } + }, + "required": [ + "totalResults", + "Resources" + ], + "title": "SCIMListResponse", + "type": "object" + }, + "SCIMMember": { + "properties": { + "display": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMMember", + "type": "object" + }, + "SCIMPatchOp": { + "properties": { + "Operations": { + "items": { + "$ref": "#/components/schemas/SCIMPatchOperation" + }, + "title": "Operations", + "type": "array" + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:api:messages:2.0:PatchOp" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + } + }, + "required": [ + "Operations" + ], + "title": "SCIMPatchOp", + "type": "object" + }, + "SCIMPatchOperation": { + "properties": { + "op": { + "title": "Op", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + }, + "value": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Value" + } + }, + "required": [ + "op" + ], + "title": "SCIMPatchOperation", + "type": "object" + }, + "SCIMServiceProviderConfig": { + "properties": { + "authenticationSchemes": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Authenticationschemes" + }, + "bulk": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "changePassword": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "etag": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "filter": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "patch": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": true + } + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "sort": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + } + }, + "title": "SCIMServiceProviderConfig", + "type": "object" + }, + "SCIMUser": { + "properties": { + "active": { + "default": true, + "title": "Active", + "type": "boolean" + }, + "displayName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Displayname" + }, + "emails": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUserEmail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Emails" + }, + "externalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Externalid" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUserGroup" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Groups" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/SCIMUserName" + }, + { + "type": "null" + } + ] + }, + "schemas": { + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "userName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Username" + } + }, + "required": [ + "schemas" + ], + "title": "SCIMUser", + "type": "object" + }, + "SCIMUserEmail": { + "properties": { + "primary": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Primary" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + }, + "value": { + "format": "email", + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMUserEmail", + "type": "object" + }, + "SCIMUserGroup": { + "properties": { + "display": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "direct", + "title": "Type" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMUserGroup", + "type": "object" + }, + "SCIMUserName": { + "properties": { + "familyName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Familyname" + }, + "formatted": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Formatted" + }, + "givenName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Givenname" + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Honorificprefix" + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Honorificsuffix" + }, + "middleName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Middlename" + } + }, + "title": "SCIMUserName", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/scim/v2": { + "get": { + "description": "Base SCIM v2 endpoint for resource discovery per RFC 7644 Section 4.\n\nReturns a ListResponse of ResourceTypes supported by this SCIM service provider.\nIdentity providers (Okta, Azure AD, etc.) use this endpoint for resource discovery.", + "operationId": "get_scim_base_scim_v2_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Scim Base", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Groups": { + "get": { + "description": "Get a list of groups according to SCIM v2 protocol", + "operationId": "get_groups_scim_v2_Groups_get", + "parameters": [ + { + "in": "query", + "name": "startIndex", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Startindex", + "type": "integer" + } + }, + { + "in": "query", + "name": "count", + "required": false, + "schema": { + "default": 10, + "maximum": 100, + "minimum": 1, + "title": "Count", + "type": "integer" + } + }, + { + "in": "query", + "name": "filter", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Filter" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Groups", + "tags": [ + "scim" + ] + }, + "post": { + "description": "Create a group according to SCIM v2 protocol", + "operationId": "create_group_scim_v2_Groups_post", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Group", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Groups/{group_id}": { + "delete": { + "description": "Delete a group according to SCIM v2 protocol", + "operationId": "delete_group_scim_v2_Groups__group_id__delete", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Group", + "tags": [ + "scim" + ] + }, + "get": { + "description": "Get a single group by ID according to SCIM v2 protocol", + "operationId": "get_group_scim_v2_Groups__group_id__get", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Group", + "tags": [ + "scim" + ] + }, + "patch": { + "description": "Patch a group according to SCIM v2 protocol", + "operationId": "patch_group_scim_v2_Groups__group_id__patch", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMPatchOp" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Group", + "tags": [ + "scim" + ] + }, + "put": { + "description": "Update a group according to SCIM v2 protocol", + "operationId": "update_group_scim_v2_Groups__group_id__put", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Group", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ResourceTypes": { + "get": { + "description": "SCIM ResourceTypes endpoint per RFC 7644 Section 4.\n\nReturns a ListResponse of all resource types supported by this service provider.", + "operationId": "get_resource_types_scim_v2_ResourceTypes_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resource Types", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ResourceTypes/{resource_type_id}": { + "get": { + "description": "Get a single ResourceType by ID per RFC 7644.", + "operationId": "get_resource_type_scim_v2_ResourceTypes__resource_type_id__get", + "parameters": [ + { + "in": "path", + "name": "resource_type_id", + "required": true, + "schema": { + "title": "ResourceType ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resource Type", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Schemas": { + "get": { + "description": "SCIM Schemas endpoint per RFC 7643 Section 7.\n\nReturns a ListResponse of all schemas supported by this service provider.", + "operationId": "get_schemas_scim_v2_Schemas_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Schemas", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Schemas/{schema_id}": { + "get": { + "description": "Get a single Schema by its URI per RFC 7643 Section 7.", + "operationId": "get_schema_scim_v2_Schemas__schema_id__get", + "parameters": [ + { + "in": "path", + "name": "schema_id", + "required": true, + "schema": { + "title": "Schema URI", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Schema", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ServiceProviderConfig": { + "get": { + "description": "Return SCIM Service Provider Configuration.", + "operationId": "get_service_provider_config_scim_v2_ServiceProviderConfig_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMServiceProviderConfig" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Service Provider Config", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Users": { + "get": { + "description": "Get a list of users according to SCIM v2 protocol", + "operationId": "get_users_scim_v2_Users_get", + "parameters": [ + { + "in": "query", + "name": "startIndex", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Startindex", + "type": "integer" + } + }, + { + "in": "query", + "name": "count", + "required": false, + "schema": { + "default": 10, + "maximum": 100, + "minimum": 1, + "title": "Count", + "type": "integer" + } + }, + { + "in": "query", + "name": "filter", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Filter" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Users", + "tags": [ + "scim" + ] + }, + "post": { + "description": "Create a user according to SCIM v2 protocol", + "operationId": "create_user_scim_v2_Users_post", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create User", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Users/{user_id}": { + "delete": { + "description": "Delete a user according to SCIM v2 protocol", + "operationId": "delete_user_scim_v2_Users__user_id__delete", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete User", + "tags": [ + "scim" + ] + }, + "get": { + "description": "Get a single user by ID according to SCIM v2 protocol", + "operationId": "get_user_scim_v2_Users__user_id__get", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get User", + "tags": [ + "scim" + ] + }, + "patch": { + "description": "Patch a user according to SCIM v2 protocol", + "operationId": "patch_user_scim_v2_Users__user_id__patch", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMPatchOp" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch User", + "tags": [ + "scim" + ] + }, + "put": { + "description": "Update a user according to SCIM v2 protocol (full replacement)", + "operationId": "update_user_scim_v2_Users__user_id__put", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update User", + "tags": [ + "scim" + ] + } + } + } + }, + "search_tools": { + "components": { + "schemas": { + "CreateSearchToolRequest": { + "properties": { + "search_tool": { + "$ref": "#/components/schemas/SearchTool" + } + }, + "required": [ + "search_tool" + ], + "title": "CreateSearchToolRequest", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListSearchToolsResponse": { + "description": "Response model for listing search tools.", + "properties": { + "search_tools": { + "items": { + "$ref": "#/components/schemas/SearchToolInfoResponse" + }, + "title": "Search Tools", + "type": "array" + } + }, + "required": [ + "search_tools" + ], + "title": "ListSearchToolsResponse", + "type": "object" + }, + "SearchTool": { + "description": "Search tool configuration.\n\nExample:\n {\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n }\n }", + "properties": { + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "litellm_params": { + "$ref": "#/components/schemas/SearchToolLiteLLMParams" + }, + "search_tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search Tool Id" + }, + "search_tool_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Search Tool Info" + }, + "search_tool_name": { + "title": "Search Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "search_tool_name", + "litellm_params" + ], + "title": "SearchTool", + "type": "object" + }, + "SearchToolInfoResponse": { + "description": "Response model for search tool information.", + "properties": { + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "is_from_config": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Is From Config" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "search_tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search Tool Id" + }, + "search_tool_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Search Tool Info" + }, + "search_tool_name": { + "title": "Search Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "title": "SearchToolInfoResponse", + "type": "object" + }, + "SearchToolLiteLLMParams": { + "description": "LiteLLM params for search tools configuration.", + "properties": { + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Base" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + }, + "max_retries": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Retries" + }, + "search_provider": { + "title": "Search Provider", + "type": "string" + }, + "timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Timeout" + } + }, + "required": [ + "search_provider" + ], + "title": "SearchToolLiteLLMParams", + "type": "object" + }, + "TestSearchToolConnectionRequest": { + "properties": { + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + } + }, + "required": [ + "litellm_params" + ], + "title": "TestSearchToolConnectionRequest", + "type": "object" + }, + "UpdateSearchToolRequest": { + "properties": { + "search_tool": { + "$ref": "#/components/schemas/SearchTool" + } + }, + "required": [ + "search_tool" + ], + "title": "UpdateSearchToolRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/search_tools": { + "post": { + "description": "Create a new search tool.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/search_tools\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"search_tool\": {\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "create_search_tool_search_tools_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateSearchToolRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Search Tool", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/list": { + "get": { + "description": "List all search tools that are available in the database and config file.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"search_tools\": [\n {\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-***\",\n \"api_base\": \"https://api.perplexity.ai\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"is_from_config\": false\n },\n {\n \"search_tool_name\": \"config-search-tool\",\n \"litellm_params\": {\n \"search_provider\": \"tavily\",\n \"api_key\": \"tvly-***\"\n },\n \"is_from_config\": true\n }\n ]\n}\n```", + "operationId": "list_search_tools_search_tools_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSearchToolsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Search Tools", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/test_connection": { + "post": { + "description": "Test connection to a search provider with the given configuration.\n\nMakes a simple test search query to verify the API key and configuration are valid.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/search_tools/test_connection\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n }\n }'\n```\n\nExample Response (Success):\n```json\n{\n \"status\": \"success\",\n \"message\": \"Successfully connected to perplexity search provider\",\n \"test_query\": \"test\",\n \"results_count\": 5\n}\n```\n\nExample Response (Failure):\n```json\n{\n \"status\": \"error\",\n \"message\": \"Authentication failed: Invalid API key\",\n \"error_type\": \"AuthenticationError\"\n}\n```", + "operationId": "test_search_tool_connection_search_tools_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestSearchToolConnectionRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Search Tool Connection", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/ui/available_providers": { + "get": { + "description": "Get the list of available search providers with their configuration fields.\n\nAuto-discovers search providers and their UI-friendly names from transformation configs.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/ui/available_providers\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"providers\": [\n {\n \"provider_name\": \"perplexity\",\n \"ui_friendly_name\": \"Perplexity\"\n },\n {\n \"provider_name\": \"tavily\",\n \"ui_friendly_name\": \"Tavily\"\n }\n ]\n}\n```", + "operationId": "get_available_search_providers_search_tools_ui_available_providers_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Available Search Providers", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/{search_tool_id}": { + "delete": { + "description": "Delete a search tool.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Search tool 123e4567-e89b-12d3-a456-426614174000 deleted successfully\",\n \"search_tool_name\": \"litellm-search\"\n}\n```", + "operationId": "delete_search_tool_search_tools__search_tool_id__delete", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Search Tool", + "tags": [ + "search_tools" + ] + }, + "get": { + "description": "Get detailed information about a specific search tool by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-***\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_search_tool_info_search_tools__search_tool_id__get", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Search Tool Info", + "tags": [ + "search_tools" + ] + }, + "put": { + "description": "Update an existing search tool.\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"search_tool\": {\n \"search_tool_name\": \"updated-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-new-key\"\n },\n \"search_tool_info\": {\n \"description\": \"Updated search tool\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"updated-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-new-key\"\n },\n \"search_tool_info\": {\n \"description\": \"Updated search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n}\n```", + "operationId": "update_search_tool_search_tools__search_tool_id__put", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateSearchToolRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Search Tool", + "tags": [ + "search_tools" + ] + } + } + } + }, + "tools": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_ToolTableRow": { + "properties": { + "assignments": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Assignments" + }, + "call_count": { + "default": 0, + "title": "Call Count", + "type": "integer" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "input_policy": { + "default": "untrusted", + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "title": "Input Policy", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "last_used_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Used At" + }, + "origin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Origin" + }, + "output_policy": { + "default": "untrusted", + "enum": [ + "trusted", + "untrusted" + ], + "title": "Output Policy", + "type": "string" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + }, + "user_agent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Agent" + } + }, + "required": [ + "tool_id", + "tool_name" + ], + "title": "LiteLLM_ToolTableRow", + "type": "object" + }, + "ToolDetailResponse": { + "properties": { + "overrides": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOverrideRow" + }, + "title": "Overrides", + "type": "array" + }, + "tool": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + } + }, + "required": [ + "tool" + ], + "title": "ToolDetailResponse", + "type": "object" + }, + "ToolListResponse": { + "properties": { + "tools": { + "items": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + }, + "title": "Tools", + "type": "array" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "tools", + "total" + ], + "title": "ToolListResponse", + "type": "object" + }, + "ToolPolicyOption": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value", + "label", + "description" + ], + "title": "ToolPolicyOption", + "type": "object" + }, + "ToolPolicyOptionsResponse": { + "properties": { + "input_policies": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOption" + }, + "title": "Input Policies", + "type": "array" + }, + "output_policies": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOption" + }, + "title": "Output Policies", + "type": "array" + } + }, + "required": [ + "input_policies", + "output_policies" + ], + "title": "ToolPolicyOptionsResponse", + "type": "object" + }, + "ToolPolicyOverrideRow": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "input_policy": { + "default": "blocked", + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "title": "Input Policy", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "override_id": { + "title": "Override Id", + "type": "string" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "override_id", + "tool_name" + ], + "title": "ToolPolicyOverrideRow", + "type": "object" + }, + "ToolPolicyUpdateRequest": { + "properties": { + "input_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "output_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Policy" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "tool_name" + ], + "title": "ToolPolicyUpdateRequest", + "type": "object" + }, + "ToolPolicyUpdateResponse": { + "properties": { + "input_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "output_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Policy" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "boolean" + } + }, + "required": [ + "tool_name", + "updated" + ], + "title": "ToolPolicyUpdateResponse", + "type": "object" + }, + "ToolUsageLogEntry": { + "description": "One spend log row for a tool call (for UI \"recent logs\" table).", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "input_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Snippet" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "spend": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Spend" + }, + "timestamp": { + "title": "Timestamp", + "type": "string" + }, + "total_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Tokens" + } + }, + "required": [ + "id", + "timestamp" + ], + "title": "ToolUsageLogEntry", + "type": "object" + }, + "ToolUsageLogsResponse": { + "properties": { + "logs": { + "items": { + "$ref": "#/components/schemas/ToolUsageLogEntry" + }, + "title": "Logs", + "type": "array" + }, + "page": { + "title": "Page", + "type": "integer" + }, + "page_size": { + "title": "Page Size", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "logs", + "total", + "page", + "page_size" + ], + "title": "ToolUsageLogsResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/toolset/{toolset_name}/mcp": { + "delete": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "get": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "head": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "options": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "patch": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "post": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "put": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/list": { + "get": { + "description": "List all auto-discovered tools and their policies.\n\nParameters:\n- input_policy: Optional filter \u2014 one of \"trusted\", \"untrusted\", \"blocked\"", + "operationId": "list_tools_v1_tool_list_get", + "parameters": [ + { + "in": "query", + "name": "input_policy", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tools", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/policy": { + "post": { + "description": "Set the input_policy and/or output_policy for a tool (global), or block for a specific team/key (override).\n\nParameters:\n- tool_name: str - The tool to update\n- input_policy: optional - \"trusted\" | \"untrusted\" | \"blocked\"\n- output_policy: optional - \"trusted\" | \"untrusted\"\n- team_id: optional - if set, create/update override for this team only\n- key_hash: optional - if set, create/update override for this key only", + "operationId": "update_tool_policy_v1_tool_policy_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyUpdateResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Tool Policy", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/policy/options": { + "get": { + "description": "Return the available input and output policy options with descriptions.\nStatic data \u2014 no DB call.", + "operationId": "get_tool_policy_options_v1_tool_policy_options_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyOptionsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Policy Options", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}": { + "get": { + "description": "Get details for a single tool.", + "operationId": "get_tool_v1_tool__tool_name__get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/detail": { + "get": { + "description": "Get a single tool with its policy overrides (for UI detail view).", + "operationId": "get_tool_detail_v1_tool__tool_name__detail_get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolDetailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Detail", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/logs": { + "get": { + "description": "Return paginated spend logs for requests that used this tool (from SpendLogToolIndex).", + "operationId": "get_tool_usage_logs_v1_tool__tool_name__logs_get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "title": "Page Size", + "type": "integer" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolUsageLogsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Usage Logs", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/overrides": { + "delete": { + "description": "Remove a policy override for a tool. Specify the override by team_id or key_hash\n(exactly one required).", + "operationId": "delete_tool_policy_override_v1_tool__tool_name__overrides_delete", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + }, + { + "description": "Team ID of the override to remove", + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team ID of the override to remove", + "title": "Team Id" + } + }, + { + "description": "Key hash of the override to remove", + "in": "query", + "name": "key_hash", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key hash of the override to remove", + "title": "Key Hash" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Tool Policy Override", + "tags": [ + "tools" + ] + } + } + } + }, + "usage_ai": { + "components": { + "schemas": { + "ChatMessage": { + "properties": { + "content": { + "title": "Content", + "type": "string" + }, + "role": { + "enum": [ + "user", + "assistant" + ], + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatMessage", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "UsageAIChatRequest": { + "properties": { + "messages": { + "description": "Chat messages (user/assistant history)", + "items": { + "$ref": "#/components/schemas/ChatMessage" + }, + "title": "Messages", + "type": "array" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model to use for AI chat", + "title": "Model" + } + }, + "required": [ + "messages" + ], + "title": "UsageAIChatRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/usage/ai/chat": { + "post": { + "description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.", + "operationId": "usage_ai_chat_usage_ai_chat_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageAIChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Usage Ai Chat", + "tags": [ + "usage_ai" + ] + } + } + } + }, + "vantage": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + }, + "VantageDryRunRequest": { + "description": "Request model for Vantage dry-run operations (capped for preview)", + "properties": { + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 500, + "description": "Limit on number of records to preview (default: 500)", + "title": "Limit" + } + }, + "title": "VantageDryRunRequest", + "type": "object" + }, + "VantageExportRequest": { + "description": "Request model for Vantage export operations (actual export, no default limit)", + "properties": { + "end_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "End time for data export in UTC", + "title": "End Time Utc" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Optional limit on number of records to export (default: no limit)", + "title": "Limit" + }, + "start_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Start time for data export in UTC", + "title": "Start Time Utc" + } + }, + "title": "VantageExportRequest", + "type": "object" + }, + "VantageExportResponse": { + "description": "Response model for Vantage export operations", + "properties": { + "dry_run_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dry run data including usage data and FOCUS transformed data", + "title": "Dry Run Data" + }, + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Summary statistics for dry run", + "title": "Summary" + } + }, + "required": [ + "message", + "status" + ], + "title": "VantageExportResponse", + "type": "object" + }, + "VantageInitRequest": { + "description": "Request model for initializing Vantage settings", + "properties": { + "api_key": { + "description": "Vantage API key for authentication", + "title": "Api Key", + "type": "string" + }, + "base_url": { + "default": "https://api.vantage.sh", + "description": "Vantage API base URL (default: https://api.vantage.sh)", + "title": "Base Url", + "type": "string" + }, + "integration_token": { + "description": "Vantage integration token for the cost-import endpoint", + "title": "Integration Token", + "type": "string" + } + }, + "required": [ + "api_key", + "integration_token" + ], + "title": "VantageInitRequest", + "type": "object" + }, + "VantageInitResponse": { + "description": "Response model for Vantage initialization", + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "title": "VantageInitResponse", + "type": "object" + }, + "VantageSettingsUpdate": { + "description": "Request model for updating Vantage settings", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage API key for authentication", + "title": "Api Key" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage API base URL", + "title": "Base Url" + }, + "integration_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage integration token", + "title": "Integration Token" + } + }, + "title": "VantageSettingsUpdate", + "type": "object" + }, + "VantageSettingsView": { + "description": "Response model for viewing Vantage settings with masked API key", + "properties": { + "api_key_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked API key showing only first 4 and last 4 characters", + "title": "Api Key Masked" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Vantage API base URL", + "title": "Base Url" + }, + "integration_token_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked integration token showing only first 4 and last 4 characters", + "title": "Integration Token Masked" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configuration status", + "title": "Status" + } + }, + "title": "VantageSettingsView", + "type": "object" + } + } + }, + "paths": { + "/vantage/delete": { + "delete": { + "description": "Delete Vantage settings from the database.\n\nOnly admin users can delete Vantage settings.", + "operationId": "delete_vantage_settings_vantage_delete_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Vantage Settings", + "tags": [ + "vantage" + ] + } + }, + "/vantage/dry-run": { + "post": { + "description": "Perform a dry run export using the Vantage logger.\n\nReturns the data that would be exported without actually sending it to Vantage.\n\nParameters:\n- limit: Limit on number of records to preview (default: 500)\n\nOnly admin users can perform Vantage exports.", + "operationId": "vantage_dry_run_export_vantage_dry_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageDryRunRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vantage Dry Run Export", + "tags": [ + "vantage" + ] + } + }, + "/vantage/export": { + "post": { + "description": "Perform an actual export using the Vantage logger.\n\nExports usage data in FOCUS CSV format to the Vantage API.\n\nParameters:\n- limit: Optional limit on number of records to export\n- start_time_utc: Optional start time for data export\n- end_time_utc: Optional end time for data export\n\nOnly admin users can perform Vantage exports.", + "operationId": "vantage_export_vantage_export_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vantage Export", + "tags": [ + "vantage" + ] + } + }, + "/vantage/init": { + "post": { + "description": "Initialize Vantage settings and store in the database.\n\nParameters:\n- api_key: Vantage API key for authentication\n- integration_token: Vantage integration token for the cost-import endpoint\n- base_url: Vantage API base URL (default: https://api.vantage.sh)\n\nOnly admin users can configure Vantage settings.", + "operationId": "init_vantage_settings_vantage_init_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Init Vantage Settings", + "tags": [ + "vantage" + ] + } + }, + "/vantage/settings": { + "get": { + "description": "View current Vantage settings.\n\nReturns the current Vantage configuration with the API key masked for security.\nOnly admin users can view Vantage settings.", + "operationId": "get_vantage_settings_vantage_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageSettingsView" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Vantage Settings", + "tags": [ + "vantage" + ] + }, + "put": { + "description": "Update existing Vantage settings.\n\nAllows updating individual Vantage configuration fields without requiring all fields.\nOnly admin users can update Vantage settings.", + "operationId": "update_vantage_settings_vantage_settings_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageSettingsUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Vantage Settings", + "tags": [ + "vantage" + ] + } + } + } + }, + "vector_store_files": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_v1_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_v1_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_v1_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_v1_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_v1_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_v1_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_create_v1_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_v1_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_v1_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_update_v1_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_v1_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_v1_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_create_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_update_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_store_files" + ] + } + } + } + }, + "vector_store_management": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_ManagedVectorStore": { + "description": "LiteLLM managed vector store object - this is is the object stored in the database", + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "custom_llm_provider": { + "title": "Custom Llm Provider", + "type": "string" + }, + "litellm_credential_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm Credential Name" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "title": "LiteLLM_ManagedVectorStore", + "type": "object" + }, + "LiteLLM_ManagedVectorStoreListResponse": { + "description": "Response format for listing vector stores", + "properties": { + "current_page": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Current Page" + }, + "data": { + "items": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStore" + }, + "title": "Data", + "type": "array" + }, + "object": { + "const": "list", + "title": "Object", + "type": "string" + }, + "total_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Count" + }, + "total_pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Pages" + } + }, + "title": "LiteLLM_ManagedVectorStoreListResponse", + "type": "object" + }, + "LiteLLM_ManagedVectorStoresTable": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "custom_llm_provider": { + "title": "Custom Llm Provider", + "type": "string" + }, + "litellm_credential_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm Credential Name" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "required": [ + "vector_store_id", + "custom_llm_provider", + "vector_store_name", + "vector_store_description", + "vector_store_metadata", + "created_at", + "updated_at", + "litellm_credential_name", + "litellm_params", + "team_id", + "user_id" + ], + "title": "LiteLLM_ManagedVectorStoresTable", + "type": "object" + }, + "ResponseLiteLLM_ManagedVectorStore": { + "properties": { + "vector_store": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoresTable" + } + }, + "title": "ResponseLiteLLM_ManagedVectorStore", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + }, + "VectorStoreDeleteRequest": { + "properties": { + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreDeleteRequest", + "type": "object" + }, + "VectorStoreInfoRequest": { + "properties": { + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreInfoRequest", + "type": "object" + }, + "VectorStoreUpdateRequest": { + "properties": { + "custom_llm_provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Custom Llm Provider" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreUpdateRequest", + "type": "object" + } + } + }, + "paths": { + "/v1/vector_store/list": { + "get": { + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "operationId": "list_vector_stores_v1_vector_store_list_get", + "parameters": [ + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 100, + "title": "Page Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoreListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Vector Stores", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/delete": { + "post": { + "description": "Delete a vector store from both database and in-memory registry.\n\nParameters:\n- vector_store_id: str - ID of the vector store to delete", + "operationId": "delete_vector_store_vector_store_delete_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreDeleteRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Vector Store", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/info": { + "post": { + "description": "Return a single vector store's details", + "operationId": "get_vector_store_info_vector_store_info_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreInfoRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResponseLiteLLM_ManagedVectorStore" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Vector Store Info", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/list": { + "get": { + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "operationId": "list_vector_stores_vector_store_list_get", + "parameters": [ + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 100, + "title": "Page Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoreListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Vector Stores", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/new": { + "post": { + "description": "Create a new vector store.\n\nParameters:\n- vector_store_id: str - Unique identifier for the vector store\n- custom_llm_provider: str - Provider of the vector store\n- vector_store_name: Optional[str] - Name of the vector store\n- vector_store_description: Optional[str] - Description of the vector store\n- vector_store_metadata: Optional[Dict] - Additional metadata for the vector store", + "operationId": "new_vector_store_vector_store_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStore" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "New Vector Store", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/update": { + "post": { + "description": "Update vector store details in both database and in-memory registry.\nThe updated data is immediately synchronized to the in-memory registry.", + "operationId": "update_vector_store_vector_store_update_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Vector Store", + "tags": [ + "vector_store_management" + ] + } + } + } + }, + "vector_stores": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "IndexCreateLiteLLMParams": { + "properties": { + "vector_store_index": { + "title": "Vector Store Index", + "type": "string" + }, + "vector_store_name": { + "title": "Vector Store Name", + "type": "string" + } + }, + "required": [ + "vector_store_index", + "vector_store_name" + ], + "title": "IndexCreateLiteLLMParams", + "type": "object" + }, + "IndexCreateRequest": { + "properties": { + "index_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Index Info" + }, + "index_name": { + "title": "Index Name", + "type": "string" + }, + "litellm_params": { + "$ref": "#/components/schemas/IndexCreateLiteLLMParams" + } + }, + "required": [ + "index_name", + "litellm_params" + ], + "title": "IndexCreateRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/indexes": { + "post": { + "description": "Create an index. Just writes the index to the database.\n\n```bash\ncurl -L -X POST 'http://0.0.0.0:4000/indexes/create' -H 'Content-Type: application/json' -H 'Authorization: Bearer sk-1234' -H 'LiteLLM-Beta: indexes_beta=v1' -d '{ \n \"index_name\": \"dall-e-3\",\n \"vector_store_index\": \"real-index-name\",\n \"vector_store_name\": \"azure-ai-search\"\n }'\n```", + "operationId": "index_create_v1_indexes_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IndexCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Index Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_v1_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_v1_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_v1_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_v1_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_v1_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_v1_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_create_v1_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_v1_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_v1_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_update_v1_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_v1_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_v1_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_create_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_update_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_stores" + ] + } + } + } + } +} diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py new file mode 100644 index 0000000000..315f6a9742 --- /dev/null +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -0,0 +1,70 @@ +""" +Per-feature OpenAPI snapshot for lazy-loaded routers. + +The committed JSON is generated by `python -m litellm.proxy._lazy_openapi_snapshot` +and consumed at runtime so /openapi.json can show full route info for unloaded +features without importing them. CI verifies the file is current and surfaces +any drift as a neutral check. +""" + +import json +import sys +from pathlib import Path +from typing import Dict, Optional + +SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json" + + +def load_snapshot() -> Optional[Dict[str, Dict]]: + if not SNAPSHOT_FILE.exists(): + return None + try: + with SNAPSHOT_FILE.open() as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return None + + +def generate_snapshot() -> Dict[str, Dict]: + import importlib + + from fastapi.openapi.utils import get_openapi + + from litellm.proxy._lazy_features import LAZY_FEATURES + from litellm.proxy.proxy_server import app + + for feat in LAZY_FEATURES: + if feat.module_path in sys.modules: + continue + try: + module = importlib.import_module(feat.module_path) + feat.register_fn(app, module) + except Exception as exc: + sys.stderr.write(f"warning: skip {feat.name}: {exc}\n") + + fragments: Dict[str, Dict] = {} + for feat in LAZY_FEATURES: + feat_routes = [ + r + for r in app.routes + if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes) + ] + if not feat_routes: + continue + full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + # Group all of a feature's routes under one tag. + for path_ops in full.get("paths", {}).values(): + for op in path_ops.values(): + if isinstance(op, dict): + op["tags"] = [feat.name] + fragments[feat.name] = { + "paths": full.get("paths", {}), + "components": {"schemas": full.get("components", {}).get("schemas", {})}, + } + return fragments + + +if __name__ == "__main__": + fragments = generate_snapshot() + SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n") + sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n") diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 92c920ca59..8532099691 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -17,6 +17,9 @@ from typing_extensions import Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_no_callback_env_reference, +) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.llms.openai import ( AllMessageValues, @@ -904,6 +907,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): agents: Optional[List[str]] = None agent_access_groups: Optional[List[str]] = None models: Optional[List[str]] = None + search_tools: Optional[List[str]] = None class BudgetLimitEntry(LiteLLMPydanticObjectBase): @@ -1868,8 +1872,10 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): raise ValueError( f"Invalid callback variable: {key}. Must be one of {valid_keys}" ) - if not isinstance(value, str): - callback_vars[key] = str(value) + callback_vars[key] = str(value) + validate_no_callback_env_reference( + key, callback_vars[key], source="key/team callback metadata" + ) return values @@ -1934,6 +1940,7 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): agent_access_groups: Optional[List[str]] = [] mcp_toolsets: Optional[List[str]] = None blocked_tools: Optional[List[str]] = [] + search_tools: Optional[List[str]] = [] class LiteLLM_TeamTable(TeamBase): @@ -2154,8 +2161,8 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.", ) auth: bool = Field( - default=False, - description="Whether authentication is required for the pass-through endpoint. If True, requests to the endpoint will require a valid LiteLLM API key.", + default=True, + description="Whether authentication is required for the pass-through endpoint. Defaults to True so a pass-through silently created without an explicit value still requires a valid LiteLLM API key — set to False only if the endpoint is meant to be a public forwarder (e.g. an unauthenticated webhook target).", ) guardrails: Optional[PassThroughGuardrailsConfig] = Field( default=None, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 840f64cfed..65638ed6c1 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -374,7 +374,7 @@ def _guardrail_modification_check( coerced = _coerce_to_dict(container) if coerced is None: return False - return any(coerced.get(key) for key in _GUARDRAIL_MODIFICATION_KEYS) + return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS) # Check both metadata keys — callers can populate either depending on the # endpoint. Cover the top-level too so root-level injection is rejected. @@ -915,7 +915,8 @@ async def get_team_member_default_budget( Fetches the team-level default per-member budget referenced by team.metadata["team_member_budget_id"]. This budget is applied to team members whose TeamMembership row has no - linked budget. Results are cached for performance. + linked budget, or whose linked budget has max_budget=NULL. Results are + cached for performance. Args: budget_id: The budget_id pulled from team.metadata["team_member_budget_id"] @@ -2962,6 +2963,116 @@ async def can_user_call_model( ) +def _search_tool_names_from_object_permission( + object_permission: Optional[LiteLLM_ObjectPermissionTable], +) -> List[str]: + """Return allowlisted search tool names from object_permission (empty = unrestricted).""" + if object_permission is None: + return [] + raw = object_permission.search_tools + if not raw: + return [] + return list(raw) + + +def _can_object_call_search_tool( + search_tool_name: str, + allowed_search_tools: List[str], + object_type: Literal["key", "team", "project"], +) -> Literal[True]: + """ + Check if an object (key/team/project) can access a specific search tool. + + Similar to _can_object_call_model but for search tools. + + Args: + search_tool_name: The search tool being requested + allowed_search_tools: List of allowed search tool names for this object + object_type: Type of object for error messaging + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + # Empty list means all search tools are allowed + if not allowed_search_tools: + return True + + # Check if the search tool is in the allowlist + if search_tool_name in allowed_search_tools: + return True + + # Access denied + raise ProxyException( + message=f"{object_type.capitalize()} not allowed to access search tool: {search_tool_name}. " + f"Allowed search tools: {allowed_search_tools}", + type=ProxyErrorTypes.key_model_access_denied, + param="search_tool_name", + code=status.HTTP_403_FORBIDDEN, + ) + + +async def can_key_call_search_tool( + search_tool_name: str, + valid_token: UserAPIKeyAuth, +) -> Literal[True]: + """ + Check if a key can access a specific search tool. + + Similar to can_key_call_model but for search tools. + + Args: + search_tool_name: The search tool being requested + valid_token: The authenticated key + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + return _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission( + valid_token.object_permission + ), + object_type="key", + ) + + +async def can_team_call_search_tool( + search_tool_name: str, + team_object: Optional[LiteLLM_TeamTable], +) -> Literal[True]: + """ + Check if a team can access a specific search tool. + + Similar to can_team_access_model but for search tools. + + Args: + search_tool_name: The search tool being requested + team_object: The team object + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + if team_object is None: + return True + + return _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission( + team_object.object_permission + ), + object_type="team", + ) + + async def is_valid_fallback_model( model: str, llm_router: Optional[Router], @@ -3293,6 +3404,7 @@ async def _check_team_member_budget( if ( team_membership is not None and team_membership.litellm_budget_table is not None + and team_membership.litellm_budget_table.max_budget is not None ): team_member_budget = team_membership.litellm_budget_table.max_budget else: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 448c975d12..91c8f2dd7c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -6,9 +6,11 @@ from typing import Any, List, Optional, Tuple from fastapi import HTTPException, Request, status +import litellm from litellm import Router, provider_list from litellm._logging import verbose_proxy_logger from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS +from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.proxy._types import * from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS @@ -53,6 +55,12 @@ def _check_valid_ip( def check_complete_credentials(request_body: dict) -> bool: """ if 'api_base' in request body. Check if complete credentials given. Prevent malicious attacks. + + Supplying an ``api_key`` is necessary but not sufficient: even with + credentials supplied, an ``api_base`` / ``base_url`` that resolves to a + private/internal/cloud-metadata address would still allow the proxy to + be used as an SSRF pivot. Validate any URL fields here so the gate + can't be bypassed with ``api_key=anything`` plus a malicious target. """ given_model: Optional[str] = None @@ -70,10 +78,27 @@ def check_complete_credentials(request_body: dict) -> bool: return False api_key_value = request_body.get("api_key") - if api_key_value and isinstance(api_key_value, str) and api_key_value.strip(): - return True + if not (api_key_value and isinstance(api_key_value, str) and api_key_value.strip()): + return False - return False + # ``validate_url`` itself doesn't consult the toggle; ``safe_get`` / + # ``async_safe_get`` do. Mirror that here so admins who explicitly + # disabled URL validation (e.g. for an internal Ollama endpoint they + # accept the SSRF risk for) aren't blocked at the proxy boundary. + if getattr(litellm, "user_url_validation", False): + for url_field in ("api_base", "base_url"): + url_value = request_body.get(url_field) + if not url_value or not isinstance(url_value, str): + continue + try: + validate_url(url_value) + except SSRFError as e: + raise ValueError( + f"Rejected request: client-side {url_field}={url_value!r} " + f"is rejected by the SSRF guard ({e})." + ) + + return True def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool: @@ -159,15 +184,42 @@ def is_request_body_safe( "aws_web_identity_token", "aws_role_name", "vertex_credentials", + # Endpoint-targeting fields that retarget the outbound request or + # an observability callback. An attacker-controlled value either + # exfiltrates the request payload (incl. messages + admin-set + # tokens) to the attacker's host, or coerces the proxy into + # authenticating against the attacker's host with admin secrets. + "aws_bedrock_runtime_endpoint", + "langsmith_base_url", + "langfuse_host", + "posthog_host", + "braintrust_host", + "slack_webhook_url", + # Provider-specific endpoint overrides that flow into the outbound + # request via ``optional_params``. Same threat as ``api_base``: + # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker + # S3; ``sagemaker_base_url`` redirects all SageMaker traffic; + # ``deployment_url`` redirects SAP deployments. + "s3_endpoint_url", + "sagemaker_base_url", + "deployment_url", ] + # The blocklist is enforced unconditionally. Legitimate clientside + # credential / endpoint passthrough goes through one of the two + # explicit admin opt-ins (``general_settings.allow_client_side_credentials`` + # proxy-wide or ``configurable_clientside_auth_params`` per deployment). + # Historically there was a third, *implicit*, *caller-controlled* path: + # ``check_complete_credentials`` returned True when the caller supplied + # any non-empty ``api_key``, which made the entire blocklist a no-op. + # That bypass turned every missing entry on the blocklist into an + # exploitable SSRF / credential-exfil hole — see GHSA-jh89-88fc-qrfp, + # GHSA-3frq-6r6h-7j64, and the chain of veria-admin findings (Dv_m860l, + # b_yRJeQ5, stN90yjP, LBlyOAc8, U2TD78kg). Removed: the blocklist now + # has a single, predictable failure mode for missing entries (a 400), + # not a credential leak. for param in banned_params: - if ( - param in request_body - and not check_complete_credentials( # allow client-credentials to be passed to proxy - request_body=request_body - ) - ): + if param in request_body: if general_settings.get("allow_client_side_credentials") is True: return True elif ( @@ -182,7 +234,10 @@ def is_request_body_safe( return True raise ValueError( f"Rejected Request: {param} is not allowed in request body. " - "Enable with `general_settings::allow_client_side_credentials` on proxy config.yaml. " + "Clientside passthrough requires explicit admin opt-in via " + "either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the " + "deployment in your proxy config.yaml. " "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b8db3cd2a7..b7700feb5b 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -21,6 +21,7 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.caching import DualCache +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.proxy._types import * @@ -472,7 +473,12 @@ async def check_api_key_for_custom_headers_or_pass_through_endpoints( for endpoint in pass_through_endpoints: if isinstance(endpoint, dict) and endpoint.get("path", "") == route: ## IF AUTH DISABLED - if endpoint.get("auth") is not True: + # Default to True: a config dict with no ``auth`` key + # otherwise produced an unauthenticated forwarder. The + # Pydantic ``PassThroughGenericEndpoint.auth`` default + # is also True, but raw config dicts skip that path — + # so this runtime check has to default to True too. + if endpoint.get("auth", True) is not True: return UserAPIKeyAuth() ## IF AUTH ENABLED ### IF CUSTOM PARSER REQUIRED @@ -1119,10 +1125,14 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) if is_master_key_valid: + # Substitute a stable alias for the raw master key so neither the + # master key nor its hash propagates into spend logs, Prometheus + # /metrics labels, audit trails, rate-limit buckets, or any other + # downstream consumer of UserAPIKeyAuth.api_key. _user_api_key_obj = await _return_user_api_key_auth_obj( user_obj=None, user_role=LitellmUserRoles.PROXY_ADMIN, - api_key=master_key, + api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, parent_otel_span=parent_otel_span, valid_token_dict={ **end_user_params, diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 160e9c23f0..935b96a0e3 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -474,6 +474,10 @@ async def retrieve_batch( # noqa: PLR0915 ) # Fix: The helper sets "file_id" but we need "batch_id" data["batch_id"] = data.pop("file_id", original_batch_id) + # Provider-config providers (e.g. bedrock) require `model` in kwargs + # so litellm.aretrieve_batch can load BedrockBatchesConfig. Without + # it the call falls into the legacy provider switch and 400s. + data["model"] = model_from_id # Retrieve batch using model credentials response = await litellm.aretrieve_batch( diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 5dcc88cacb..adf562d69c 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -313,23 +313,24 @@ sequenceDiagram participant Proxy as LiteLLM Proxy participant SSO as SSO Provider - CLI->>CLI: Generate key ID (sk-uuid) - CLI->>Browser: Open /sso/key/generate?source=litellm-cli&key=sk-uuid + CLI->>Proxy: POST /sso/cli/start + Proxy->>CLI: Return login_id, poll_secret, user_code + CLI->>Browser: Open /sso/key/generate?source=litellm-cli&key=login_id - Browser->>Proxy: GET /sso/key/generate?source=litellm-cli&key=sk-uuid - Proxy->>Proxy: Set cli_state = litellm-session-token:sk-uuid - Proxy->>SSO: Redirect with state=litellm-session-token:sk-uuid + Browser->>Proxy: GET /sso/key/generate?source=litellm-cli&key=login_id + Proxy->>Proxy: Set cli_state = litellm-session-token:login_id + Proxy->>SSO: Redirect with state=litellm-session-token:login_id SSO->>Browser: Show login page Browser->>SSO: User authenticates - SSO->>Proxy: Redirect to /sso/callback?state=litellm-session-token:sk-uuid + SSO->>Proxy: Redirect to /sso/callback?state=litellm-session-token:login_id Proxy->>Proxy: Check if state starts with "litellm-session-token:" - Proxy->>Proxy: Generate API key with ID=sk-uuid - Proxy->>Browser: Show success page + Proxy->>Browser: Prompt for user_code + Browser->>Proxy: POST /sso/cli/complete/login_id - CLI->>Proxy: Poll /sso/cli/poll/sk-uuid - Proxy->>CLI: Return {"status": "ready", "key": "sk-uuid"} + CLI->>Proxy: Poll /sso/cli/poll/login_id with poll_secret header + Proxy->>CLI: Return {"status": "ready", "key": "jwt"} CLI->>CLI: Save key to ~/.litellm/token.json ``` @@ -343,13 +344,13 @@ The CLI provides three authentication commands: ### Authentication Flow Steps -1. **Generate Session ID**: CLI generates a unique key ID (`sk-{uuid}`) -2. **Open Browser**: CLI opens browser to `/sso/key/generate` with CLI source and key parameters -3. **SSO Redirect**: Proxy sets the formatted state (`litellm-session-token:sk-uuid`) as OAuth state parameter and redirects to SSO provider +1. **Start Session**: CLI creates a short-lived login session with `/sso/cli/start` +2. **Open Browser**: CLI opens browser to `/sso/key/generate` with CLI source and login ID parameters +3. **SSO Redirect**: Proxy sets the formatted state (`litellm-session-token:{login_id}`) as OAuth state parameter and redirects to SSO provider 4. **User Authentication**: User completes SSO authentication in browser 5. **Callback Processing**: SSO provider redirects back to proxy with state parameter -6. **Key Generation**: Proxy detects CLI login (state starts with "litellm-session-token:") and generates API key with pre-specified ID -7. **Polling**: CLI polls `/sso/cli/poll/{key_id}` endpoint until key is ready +6. **User Code Verification**: Browser confirms the verification code shown in the CLI +7. **Polling**: CLI polls `/sso/cli/poll/{login_id}` with the polling secret header until the JWT is ready 8. **Token Storage**: CLI saves the authentication token to `~/.litellm/token.json` ### Benefits of This Approach @@ -357,7 +358,7 @@ The CLI provides three authentication commands: - **No Local Server**: No need to run a local callback server - **Standard OAuth**: Uses OAuth 2.0 state parameter correctly - **Remote Compatible**: Works with remote proxy servers -- **Secure**: Uses UUID session identifiers +- **Secure**: Keeps the polling secret out of the browser handoff - **Simple Setup**: No additional OAuth redirect URL configuration needed ### Token Storage diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index aeb59e78a5..a9ea7a84e1 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -5,6 +5,7 @@ import time import webbrowser from pathlib import Path from typing import Any, Dict, List, Optional +from urllib.parse import urlencode import click import requests @@ -241,7 +242,7 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any def prompt_team_selection_fallback( - teams: List[Dict[str, Any]] + teams: List[Dict[str, Any]], ) -> Optional[Dict[str, Any]]: """Fallback team selection for non-interactive environments""" if not teams: @@ -279,6 +280,7 @@ def prompt_team_selection_fallback( def _poll_for_ready_data( url: str, *, + headers: Optional[Dict[str, str]] = None, total_timeout: int = 300, poll_interval: int = 2, request_timeout: int = 10, @@ -291,7 +293,10 @@ def _poll_for_ready_data( ) -> Optional[Dict[str, Any]]: for attempt in range(total_timeout // poll_interval): try: - response = requests.get(url, timeout=request_timeout) + request_kwargs: Dict[str, Any] = {"timeout": request_timeout} + if headers is not None: + request_kwargs["headers"] = headers + response = requests.get(url, **request_kwargs) if response.status_code == 200: data = response.json() status = data.get("status") @@ -346,7 +351,23 @@ def _normalize_teams(teams, team_details): return [] -def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: +def _start_cli_sso_flow(base_url: str) -> Dict[str, Any]: + response = requests.post(f"{base_url}/sso/cli/start", timeout=10) + response.raise_for_status() + data = response.json() + required_fields = ("login_id", "poll_secret", "user_code") + if not all(isinstance(data.get(field), str) for field in required_fields): + raise ValueError("Invalid CLI SSO start response") + return data + + +def _get_cli_sso_poll_headers(poll_secret: str) -> Dict[str, str]: + return {"x-litellm-cli-poll-secret": poll_secret} + + +def _poll_for_authentication( + base_url: str, key_id: str, poll_secret: str +) -> Optional[dict]: """ Poll the server for authentication completion and handle team selection. @@ -356,6 +377,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: poll_url = f"{base_url}/sso/cli/poll/{key_id}" data = _poll_for_ready_data( poll_url, + headers=_get_cli_sso_poll_headers(poll_secret), pending_message="Still waiting for authentication...", ) if not data: @@ -373,6 +395,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: jwt_with_team = _handle_team_selection_during_polling( base_url=base_url, key_id=key_id, + poll_secret=poll_secret, teams=normalized_teams, ) @@ -410,7 +433,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: def _handle_team_selection_during_polling( - base_url: str, key_id: str, teams: List[Dict[str, Any]] + base_url: str, key_id: str, poll_secret: str, teams: List[Dict[str, Any]] ) -> Optional[str]: """ Handle team selection and re-poll with selected team_id. @@ -441,6 +464,7 @@ def _handle_team_selection_during_polling( poll_url = f"{base_url}/sso/cli/poll/{key_id}?team_id={team_id}" data = _poll_for_ready_data( poll_url, + headers=_get_cli_sso_poll_headers(poll_secret), pending_message="Still waiting for team authentication...", other_status_message="Waiting for team authentication to complete...", http_error_log_every=10, @@ -514,29 +538,24 @@ def _render_and_prompt_for_team_selection(teams: List[Dict[str, Any]]) -> Option @click.pass_context def login(ctx: click.Context): """Login to LiteLLM proxy using SSO authentication""" - from litellm._uuid import uuid from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER from litellm.proxy.client.cli.interface import show_commands base_url = ctx.obj["base_url"] - # Check if we have an existing key to regenerate - existing_key = get_stored_api_key() - - # Generate unique key ID for this login session - key_id = f"sk-{str(uuid.uuid4())}" - try: - # Construct SSO login URL with CLI source and pre-generated key - sso_url = f"{base_url}/sso/key/generate?source={LITELLM_CLI_SOURCE_IDENTIFIER}&key={key_id}" + cli_sso_flow = _start_cli_sso_flow(base_url=base_url) + key_id = cli_sso_flow["login_id"] + poll_secret = cli_sso_flow["poll_secret"] + user_code = cli_sso_flow["user_code"] - # If we have an existing key, include it as a parameter to the login endpoint - # The server will encode it in the OAuth state parameter for the SSO flow - if existing_key: - sso_url += f"&existing_key={existing_key}" + sso_url = f"{base_url}/sso/key/generate?" + urlencode( + {"source": LITELLM_CLI_SOURCE_IDENTIFIER, "key": key_id} + ) click.echo(f"Opening browser to: {sso_url}") click.echo("Please complete the SSO authentication in your browser...") + click.echo(f"Verification code: {user_code}") click.echo(f"Session ID: {key_id}") # Open browser @@ -545,7 +564,9 @@ def login(ctx: click.Context): # Poll for authentication completion click.echo("Waiting for authentication...") - auth_result = _poll_for_authentication(base_url=base_url, key_id=key_id) + auth_result = _poll_for_authentication( + base_url=base_url, key_id=key_id, poll_secret=poll_secret + ) if auth_result: api_key = auth_result["api_key"] diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c8fab4be4a..76c52f83ee 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -619,6 +619,67 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.error(f"Error setting custom headers: {e}") return {} + @staticmethod + async def build_litellm_proxy_success_headers_from_llm_response( + *, + response: Any, + request_data: dict, + request: Request, + user_api_key_dict: UserAPIKeyAuth, + logging_obj: LiteLLMLoggingObj, + version: Optional[str], + proxy_logging_obj: ProxyLogging, + ) -> Dict[str, str]: + """ + Build LiteLLM proxy response headers for routes that call the LLM directly + (e.g. Google native :generateContent) instead of base_process_llm_request. + """ + if isinstance(response, dict): + hidden_params = response.get("_hidden_params") or {} + else: + hidden_params = getattr(response, "_hidden_params", None) or {} + if not isinstance(hidden_params, dict): + hidden_params = {} + + model_id = ProxyBaseLLMRequestProcessing._get_model_id_from_response( + hidden_params, request_data + ) + + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + response_cost = hidden_params.get("response_cost", None) or "" + fastest_response_batch_completion = hidden_params.get( + "fastest_response_batch_completion", None + ) + additional_headers = hidden_params.get("additional_headers", {}) or {} + + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=logging_obj.litellm_call_id, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + fastest_response_batch_completion=fastest_response_batch_completion, + request_data=request_data, + hidden_params=hidden_params, + litellm_logging_obj=logging_obj, + **additional_headers, + ) + + callback_headers = await proxy_logging_obj.post_call_response_headers_hook( + data=request_data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) + if callback_headers: + custom_headers.update(callback_headers) + + return custom_headers + async def common_processing_pre_call_logic( self, request: Request, @@ -875,7 +936,7 @@ class ProxyBaseLLMRequestProcessing: else: verbose_proxy_logger.debug( "Request received by LiteLLM:\n%s", - json.dumps(self.data, indent=4, default=str), + _payload_str, ) async def base_process_llm_request( # noqa: PLR0915 @@ -1511,9 +1572,7 @@ class ProxyBaseLLMRequestProcessing: _response = assembled_response try: from litellm.proxy.proxy_server import llm_router as _global_llm_router - from litellm.proxy.utils import ( - _check_and_merge_model_level_guardrails, - ) + from litellm.proxy.utils import _check_and_merge_model_level_guardrails guardrail_data = _check_and_merge_model_level_guardrails( data=captured_data, llm_router=_global_llm_router @@ -1690,11 +1749,12 @@ class ProxyBaseLLMRequestProcessing: elif isinstance(e, httpx.HTTPStatusError): # Handle httpx.HTTPStatusError - extract actual error from response # This matches the original behavior before the refactor in commit 511d435f6f - error_body = await e.response.aread() + http_status_error: httpx.HTTPStatusError = e + error_body = await http_status_error.response.aread() error_text = error_body.decode("utf-8") raise HTTPException( - status_code=e.response.status_code, + status_code=http_status_error.response.status_code, detail={"error": error_text}, ) error_msg = f"{str(e)}" diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 7ddd722a80..198d9503cb 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -14,6 +14,8 @@ from litellm.types.utils import ( blue_color_code = "\033[94m" reset_color_code = "\033[0m" +TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY = "_pillar_response_headers_trusted" + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -417,10 +419,19 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]: if "semantic-similarity" in _metadata: headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"]) + is_trusted_pillar_metadata = ( + _metadata.get(TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY) is True + ) pillar_headers = _metadata.get("pillar_response_headers") - if isinstance(pillar_headers, dict): - headers.update(pillar_headers) - elif "pillar_flagged" in _metadata: + if is_trusted_pillar_metadata and isinstance(pillar_headers, dict): + headers.update( + { + key: str(value) + for key, value in pillar_headers.items() + if isinstance(key, str) and key.lower().startswith("x-pillar-") + } + ) + elif is_trusted_pillar_metadata and "pillar_flagged" in _metadata: headers["x-pillar-flagged"] = str(_metadata["pillar_flagged"]).lower() return headers diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index e486336cec..0928ce914d 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -52,6 +52,37 @@ class ResetBudgetJob: ### RESET MULTI-WINDOW BUDGETS ### await self.reset_budget_windows() + @staticmethod + async def _invalidate_spend_counter(counter_key: str) -> None: + """Zero a spend counter so a DB-row reset takes effect immediately. + + Call AFTER the DB write commits. Clearing Redis before the DB + commit opens a window where get_current_spend reads 0 from Redis + while the DB still holds the pre-reset value, allowing bypass. + """ + try: + from litellm.proxy.proxy_server import spend_counter_cache + + spend_counter_cache.in_memory_cache.set_cache( + key=counter_key, value=0.0, ttl=60 + ) + if spend_counter_cache.redis_cache is not None: + try: + await spend_counter_cache.redis_cache.async_set_cache( + key=counter_key, value=0.0, ttl=60 + ) + except Exception as redis_err: + verbose_proxy_logger.warning( + "Failed to reset spend counter %s in Redis: %s. " + "Budget may be over-enforced until counter expires.", + counter_key, + redis_err, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to reset spend counter %s: %s", counter_key, e + ) + async def reset_budget_for_litellm_team_members( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): @@ -64,46 +95,30 @@ class ResetBudgetJob: if budget.budget_id is not None ] - # Reset spend counters for affected team members. - # Reset Redis directly so a transient failure doesn't leave stale - # counters that get_current_spend would read as authoritative. try: - from litellm.proxy.proxy_server import spend_counter_cache - memberships = await self.prisma_client.db.litellm_teammembership.find_many( where={"budget_id": {"in": budget_ids}} ) - for m in memberships: - counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" - # Always reset in-memory - spend_counter_cache.in_memory_cache.set_cache( - key=counter_key, value=0.0 - ) - # Explicitly reset Redis with warning on failure - if spend_counter_cache.redis_cache is not None: - try: - await spend_counter_cache.redis_cache.async_set_cache( - key=counter_key, value=0.0 - ) - except Exception as redis_err: - verbose_proxy_logger.warning( - "Failed to reset team member spend counter in Redis %s: %s. " - "Budget may be over-enforced until counter expires.", - counter_key, - redis_err, - ) except Exception as e: + memberships = [] verbose_proxy_logger.warning( - "Failed to reset team member spend counters: %s", e + "Failed to fetch team memberships for counter invalidation: %s", e ) - return await self.prisma_client.db.litellm_teammembership.update_many( + update_result = await self.prisma_client.db.litellm_teammembership.update_many( where={"budget_id": {"in": budget_ids}}, data={ "spend": 0, }, ) + for m in memberships: + await self._invalidate_spend_counter( + f"spend:team_member:{m.user_id}:{m.team_id}" + ) + + return update_result + async def reset_budget_for_keys_linked_to_budgets( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): @@ -126,17 +141,36 @@ class ResetBudgetJob: if not budget_ids: return - return await self.prisma_client.db.litellm_verificationtoken.update_many( - where={ - "budget_id": {"in": budget_ids}, - "budget_duration": None, # only keys without their own reset schedule - "spend": {"gt": 0}, # only reset keys that have accumulated spend - }, - data={ - "spend": 0, - }, + where_clause: dict = { + "budget_id": {"in": budget_ids}, + "budget_duration": None, # only keys without their own reset schedule + "spend": {"gt": 0}, # only reset keys that have accumulated spend + } + + try: + keys = await self.prisma_client.db.litellm_verificationtoken.find_many( + where=where_clause + ) + except Exception as e: + keys = [] + verbose_proxy_logger.warning( + "Failed to fetch keys for counter invalidation: %s", e + ) + + update_result = ( + await self.prisma_client.db.litellm_verificationtoken.update_many( + where=where_clause, + data={ + "spend": 0, + }, + ) ) + for k in keys: + await self._invalidate_spend_counter(f"spend:key:{k.token}") + + return update_result + async def reset_budget_for_litellm_budget_table(self): """ Resets the budget for all LiteLLM End-Users (Customers), and Team Members if their budget has expired @@ -365,6 +399,10 @@ class ResetBudgetJob: data_list=updated_keys, table_name="key", ) + for k in updated_keys: + token = getattr(k, "token", None) + if token: + await self._invalidate_spend_counter(f"spend:key:{token}") end_time = time.time() if len(failed_keys) > 0: # If any keys failed to reset @@ -450,6 +488,12 @@ class ResetBudgetJob: data_list=updated_users, table_name="user", ) + for u in updated_users: + user_id = getattr(u, "user_id", None) + if user_id: + await self._invalidate_spend_counter( + f"spend:user:{user_id}" + ) end_time = time.time() if len(failed_users) > 0: # If any users failed to reset @@ -541,6 +585,12 @@ class ResetBudgetJob: data_list=updated_teams, table_name="team", ) + for t in updated_teams: + team_id = getattr(t, "team_id", None) + if team_id: + await self._invalidate_spend_counter( + f"spend:team:{team_id}" + ) end_time = time.time() if len(failed_teams) > 0: # If any teams failed to reset diff --git a/litellm/proxy/common_utils/static_asset_utils.py b/litellm/proxy/common_utils/static_asset_utils.py new file mode 100644 index 0000000000..c108af2b47 --- /dev/null +++ b/litellm/proxy/common_utils/static_asset_utils.py @@ -0,0 +1,52 @@ +"""Helpers for unauthenticated logo / favicon endpoints.""" + +import os +from typing import Optional, Tuple + +from litellm._logging import verbose_proxy_logger + +LOCAL_IMAGE_HEADER_BYTES = 512 + + +def detect_local_image_media_type(header: bytes) -> Optional[str]: + """Return a browser image media type for supported local image signatures.""" + if header[0:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if header[0:4] == b"GIF8" and header[5:6] == b"a": + return "image/gif" + if header[0:3] == b"\xff\xd8\xff": + return "image/jpeg" + if header[0:4] == b"RIFF" and header[8:12] == b"WEBP": + return "image/webp" + if header[0:4] in (b"\x00\x00\x01\x00", b"\x00\x00\x02\x00"): + return "image/x-icon" + return None + + +def resolve_validated_local_image_path(candidate: str) -> Optional[Tuple[str, str]]: + """Resolve ``candidate`` only when it is an existing supported image file.""" + if not candidate: + return None + try: + resolved = os.path.realpath(os.path.expanduser(candidate)) + except (OSError, ValueError): + return None + if not os.path.isfile(resolved): + return None + + try: + with open(resolved, "rb") as f: + header = f.read(LOCAL_IMAGE_HEADER_BYTES) + except OSError as exc: + verbose_proxy_logger.debug("Could not read local asset %r: %s", candidate, exc) + return None + + media_type = detect_local_image_media_type(header) + if media_type is None: + verbose_proxy_logger.warning( + "Local asset %r is not a supported image file; falling back to default.", + candidate, + ) + return None + + return resolved, media_type diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index cfa90a48ee..ab9d341aa5 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,5 +1,6 @@ -from typing import Union +from typing import Any, Awaitable, Callable, Optional, Union +from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( DB_CONNECTION_ERROR_TYPES, ProxyErrorTypes, @@ -123,3 +124,138 @@ class PrismaDBExceptionHandler: ): return None raise e + + +# Default fallback timeouts when neither the caller nor the prisma_client +# expose `_db_auth_reconnect_timeout_seconds` / `_db_auth_reconnect_lock_timeout_seconds`. +# Match the auth path's existing defaults so behavior is uniform across read paths. +_DEFAULT_RECONNECT_TIMEOUT_SECONDS = 2.0 +_DEFAULT_RECONNECT_LOCK_TIMEOUT_SECONDS = 0.1 + + +def _coerce_timeout(value: Any, fallback: float) -> float: + """Return `value` if it is a real int/float, else `fallback`. Guards + against tests that mock `prisma_client` and leave the timeout slots as + MagicMock instances.""" + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return fallback + + +async def call_with_db_reconnect_retry( + prisma_client: Any, + coro_factory: Callable[[], Awaitable[Any]], + *, + reason: str, + timeout_seconds: Optional[float] = None, + lock_timeout_seconds: Optional[float] = None, +) -> Any: + """Run a Prisma read coroutine with one transport-reconnect-and-retry. + + The canonical "self-heal a transient DB transport blip" wrapper used by + `PrismaClient.get_generic_data` and other read paths. Mirrors the inline + pattern in `auth_checks._fetch_key_object_from_db_with_reconnect` so we + have a single implementation rather than three drifting copies. + + Behavior: + 1. Await `coro_factory()`. On success, return its value. + 2. On exception, if it is NOT a transport error (per + `is_database_transport_error`), re-raise — data-layer errors like + `UniqueViolationError` mean the DB is reachable, reconnect would be + pointless. + 3. If `prisma_client` does not expose `attempt_db_reconnect`, re-raise. + This guards against partial stand-ins / older clients in tests. + 4. Call `prisma_client.attempt_db_reconnect(reason=...)`. If it returns + False (cooldown / lock contention / reconnect failure), re-raise. + 5. Otherwise await `coro_factory()` a second time and return / propagate + its result. At-most-one retry by construction — no infinite loop. + + `coro_factory` MUST be a zero-arg callable that returns a fresh awaitable + on each call. Passing an already-awaited coroutine would fail on retry + with `RuntimeError: cannot reuse already awaited coroutine`. + + `reason` should follow `___failure` so + telemetry distinguishes between fan-out callers (e.g. + `_update_config_from_db` issues four concurrent reads). + + Args: + prisma_client: The `PrismaClient` (or stand-in) that owns + `attempt_db_reconnect` and the `_db_auth_reconnect_*` defaults. + coro_factory: Zero-arg callable returning the read awaitable. + reason: Telemetry tag forwarded to `attempt_db_reconnect`. + timeout_seconds: Optional override for the reconnect cycle timeout. + Defaults to `prisma_client._db_auth_reconnect_timeout_seconds`, + then to 2.0s. + lock_timeout_seconds: Optional override for how long the helper will + wait to acquire the reconnect lock. Defaults to + `prisma_client._db_auth_reconnect_lock_timeout_seconds`, then to + 0.1s. + + Returns: + Whatever `coro_factory()` returns (on first or second attempt). + + Raises: + Whatever `coro_factory()` raises if the failure is not a transport + error, or if the reconnect attempt does not succeed, or if the retry + also fails. + """ + try: + return await coro_factory() + except Exception as first_exc: + if not PrismaDBExceptionHandler.is_database_transport_error(first_exc): + raise + if not hasattr(prisma_client, "attempt_db_reconnect"): + raise + + resolved_timeout = _coerce_timeout( + ( + timeout_seconds + if timeout_seconds is not None + else getattr(prisma_client, "_db_auth_reconnect_timeout_seconds", None) + ), + _DEFAULT_RECONNECT_TIMEOUT_SECONDS, + ) + resolved_lock_timeout = _coerce_timeout( + ( + lock_timeout_seconds + if lock_timeout_seconds is not None + else getattr( + prisma_client, "_db_auth_reconnect_lock_timeout_seconds", None + ) + ), + _DEFAULT_RECONNECT_LOCK_TIMEOUT_SECONDS, + ) + + verbose_proxy_logger.warning( + "DB transport error on read; attempting reconnect-and-retry. reason=%s error=%s", + reason, + first_exc, + ) + + # Preserve the original transport error in telemetry. If + # `attempt_db_reconnect` itself raises (e.g. lock cancellation, timer + # error, unexpected internal failure), surfacing that exception + # instead of `first_exc` would mask the actual DB transport problem + # in `failure_handler` / `db_exceptions` alerts. Chain the reconnect + # error as the cause for debuggability without losing the original. + try: + did_reconnect = await prisma_client.attempt_db_reconnect( + reason=reason, + timeout_seconds=resolved_timeout, + lock_timeout_seconds=resolved_lock_timeout, + ) + except Exception as reconnect_exc: + verbose_proxy_logger.warning( + "DB reconnect attempt raised; preserving original transport error. " + "reason=%s reconnect_error=%s", + reason, + reconnect_exc, + ) + raise first_exc from reconnect_exc + if not did_reconnect: + raise + + # At most one retry. If the retry also raises a transport error, we + # propagate — repeated reconnect-loops are the watchdog's job, not + # this helper's. + return await coro_factory() diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 73735796eb..d112e22230 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -52,18 +52,25 @@ class PrismaWrapper: engine = self._original_prisma._engine process = getattr(engine, "process", None) if engine is not None else None if process is not None: - return process.pid + pid = process.pid + if isinstance(pid, int): + return pid except (AttributeError, TypeError): pass return 0 @staticmethod async def _kill_engine_process(pid: int) -> None: - """Force-kill an orphaned engine subprocess to prevent DB connection pool leaks. + """Force-kill the engine subprocess to prevent DB connection pool leaks. - Called when disconnect() fails and the old engine process may still be - holding open connections. Sends SIGTERM for graceful shutdown, waits - briefly, then SIGKILL as a backstop. + Called on every reconnect (in `recreate_prisma_client`) to retire the + old query-engine subprocess without invoking prisma-client-py's + synchronous `disconnect()` — which blocks the asyncio event loop on + `subprocess.Popen.wait()` for 30-120+ seconds when the engine is + stuck on TCP close. + + Sends SIGTERM for graceful shutdown, waits briefly, then SIGKILL as + a backstop. """ if pid <= 0: return @@ -72,7 +79,7 @@ class PrismaWrapper: except (ProcessLookupError, PermissionError, OSError): return # Already dead or inaccessible verbose_proxy_logger.warning( - "Sent SIGTERM to orphaned prisma-query-engine PID %s after failed disconnect.", + "Sent SIGTERM to prisma-query-engine PID %s during reconnect.", pid, ) # Brief wait for graceful shutdown, then force-kill @@ -217,15 +224,18 @@ class PrismaWrapper: async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None ): - """Disconnect and reconnect the Prisma client with a new database URL.""" + """Disconnect and reconnect the Prisma client with a new database URL. + + Kills the old engine subprocess directly (SIGTERM → SIGKILL) rather than + calling `disconnect()`. prisma-client-py's `disconnect()` calls a + synchronous `subprocess.Popen.wait()` that can freeze the asyncio event + loop for 30-120+ seconds when the engine is stuck on TCP close, + breaking `/health/liveliness` and causing Kubernetes pod restarts. + """ from prisma import Prisma # type: ignore old_engine_pid = self._get_engine_pid() - - try: - await self._original_prisma.disconnect() - except Exception as e: - verbose_proxy_logger.warning(f"Failed to disconnect Prisma client: {e}") + if old_engine_pid > 0: await self._kill_engine_process(old_engine_pid) if http_client is not None: diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index bf60a087c6..a979471dc8 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -129,7 +129,9 @@ class SpendCounterReseed: """ lock = await SpendCounterReseed._get_lock(counter_key) async with lock: - # Re-check after acquiring the lock - another waiter may have warmed it. + # Re-check after acquiring the lock. Skip in-memory on a clean + # Redis miss - in-memory is per-pod-stale. + redis_clean_miss = False if spend_counter_cache.redis_cache is not None: try: val = await spend_counter_cache.redis_cache.async_get_cache( @@ -137,11 +139,13 @@ class SpendCounterReseed: ) if val is not None: return float(val) + redis_clean_miss = True except Exception: pass - val = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) - if val is not None: - return float(val) + if not redis_clean_miss: + val = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) + if val is not None: + return float(val) db_spend = await SpendCounterReseed.from_db(prisma_client, counter_key) if db_spend is None: @@ -149,7 +153,7 @@ class SpendCounterReseed: # Warm even when 0 so subsequent reads hit cache, not DB. try: await spend_counter_cache.async_increment_cache( - key=counter_key, value=db_spend + key=counter_key, value=db_spend, refresh_ttl=True ) except Exception: verbose_proxy_logger.exception( diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 9768d93e92..967ac9f0ac 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -1,10 +1,6 @@ -from datetime import datetime +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import ORJSONResponse -from fastapi import APIRouter, Depends, HTTPException, Request, Response -from fastapi.responses import ORJSONResponse, StreamingResponse - -import litellm -from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -30,11 +26,17 @@ async def google_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import ( general_settings, llm_router, proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, version, ) @@ -42,38 +44,33 @@ async def google_generate_content( if "model" not in data: data["model"] = model_name - # Extract generationConfig and pass it as config parameter - generation_config = data.pop("generationConfig", None) - if generation_config: - data["config"] = generation_config - - # Add user authentication metadata for cost tracking - data = await add_litellm_data_to_request( - data=data, - request=request, - user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config, - general_settings=general_settings, - version=version, - ) - - # Create logging object with full request metadata so callbacks (e.g. S3) get user/trace_id - data["litellm_call_id"] = request.headers.get( - "x-litellm-call-id", str(uuid.uuid4()) - ) - logging_obj, data = litellm.utils.function_setup( - original_function="agenerate_content", - rules_obj=litellm.utils.Rules(), - start_time=datetime.now(), - **data, - ) - data["litellm_logging_obj"] = logging_obj - - # call router - if llm_router is None: - raise HTTPException(status_code=500, detail="Router not initialized") - response = await llm_router.agenerate_content(**data) - return response + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="agenerate_content", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=model_name, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) @router.post( @@ -90,57 +87,52 @@ async def google_stream_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import ( general_settings, llm_router, proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, version, ) data = await _read_request_body(request=request) - if "model" not in data: data["model"] = model_name + data["stream"] = True - data["stream"] = True # enforce streaming for this endpoint - - # Extract generationConfig and pass it as config parameter - generation_config = data.pop("generationConfig", None) - if generation_config: - data["config"] = generation_config - - # Add user authentication metadata for cost tracking - data = await add_litellm_data_to_request( - data=data, - request=request, - user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config, - general_settings=general_settings, - version=version, - ) - - # Create logging object with full request metadata so streaming END callbacks (e.g. S3) get user/trace_id - data["litellm_call_id"] = request.headers.get( - "x-litellm-call-id", str(uuid.uuid4()) - ) - logging_obj, data = litellm.utils.function_setup( - original_function="agenerate_content_stream", - rules_obj=litellm.utils.Rules(), - start_time=datetime.now(), - **data, - ) - data["litellm_logging_obj"] = logging_obj - - # call router - if llm_router is None: - raise HTTPException(status_code=500, detail="Router not initialized") - response = await llm_router.agenerate_content_stream(**data) - - # Check if response is an async iterator (streaming response) - if response is not None and hasattr(response, "__aiter__"): - return StreamingResponse(content=response, media_type="text/event-stream") - return response + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="agenerate_content_stream", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=model_name, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) @router.post( diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index e5df26b7c2..bb1db3d62d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -71,6 +71,7 @@ from litellm.types.utils import ( ) GUARDRAIL_NAME = "bedrock" +_BEDROCK_DYNAMIC_BODY_DENYLIST = frozenset({"content", "source"}) class GuardrailMessageFilterResult(NamedTuple): @@ -413,11 +414,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) api_key: Optional[str] = None if request_data: - bedrock_request_data.update( + dynamic_request_body_params = ( self.get_guardrail_dynamic_request_body_params( request_data=request_data ) ) + bedrock_request_data.update( + { + key: value + for key, value in dynamic_request_body_params.items() + if key not in _BEDROCK_DYNAMIC_BODY_DENYLIST + } + ) if request_data.get("api_key") is not None: api_key = request_data["api_key"] diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 1b3f11e56f..c9c73053a0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -29,6 +29,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( + TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY, add_guardrail_to_applied_guardrails_header, get_metadata_variable_name_from_kwargs, ) @@ -144,6 +145,7 @@ def build_pillar_response_headers(metadata_store: Dict[str, Any]) -> Dict[str, s if headers: metadata_store["pillar_response_headers"] = headers + metadata_store[TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY] = True return headers diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 5cdd9ddb4b..6bb9c1f507 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -41,6 +41,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -61,9 +62,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=response.token_id or "", @@ -102,6 +105,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -117,9 +121,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=data.key, @@ -140,6 +146,7 @@ class KeyManagementEventHooks: ): from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -189,9 +196,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.token, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=existing_key_row.token, @@ -220,6 +229,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -237,9 +247,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.token, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=key.token, diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 38623f9209..08fa8d4dfa 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -192,13 +192,19 @@ class UserManagementEventHooks: if not litellm.store_audit_logs: return + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) + await create_audit_log_for_update( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.USER_TABLE_NAME, object_id=user_id, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5804e3f8d9..3077efe116 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -6,6 +6,7 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from fastapi import Request +from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm @@ -104,6 +105,112 @@ LITELLM_METADATA_ROUTES = ( "files", ) +_UNTRUSTED_ROOT_CONTROL_FIELDS = ( + "proxy_server_request", + "standard_logging_object", + "secret_fields", + "mock_response", + "mock_tool_calls", + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "applied_guardrails", + "applied_policies", + "policy_sources", + "pillar_response_headers", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", +) + +_UNTRUSTED_METADATA_CONTROL_FIELDS = ( + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "pillar_response_headers", + "_pillar_response_headers_trusted", + "pillar_flagged", + "pillar_scanners", + "pillar_evidence", + "pillar_evidence_truncated", + "pillar_session_id_response", + "applied_guardrails", + "applied_policies", + "policy_sources", + "standard_logging_object", + "proxy_server_request", + "secret_fields", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", +) + +_UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS = frozenset( + { + "litellm-disable-message-redaction", + } +) +_CLIENT_MOCK_CONTROL_FIELDS = frozenset({"mock_response", "mock_tool_calls"}) +_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY = "allow_client_mock_response" +_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY = ( + "allow_client_message_redaction_opt_out" +) + + +def _strip_untrusted_request_header_controls( + headers: Any, + *, + allow_client_message_redaction_opt_out: bool = False, +) -> None: + if not isinstance(headers, dict): + return + + for header_name in list(headers.keys()): + if ( + isinstance(header_name, str) + and header_name.lower() in _UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS + ): + if allow_client_message_redaction_opt_out: + continue + headers.pop(header_name, None) + + +def _is_false_like(value: Any) -> bool: + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return value.strip().lower() in {"false", "0", "no", "off"} + return False + + +def _key_or_team_metadata_flag_is_true( + user_api_key_dict: UserAPIKeyAuth, + metadata_key: str, +) -> bool: + for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata): + if ( + isinstance(admin_metadata, dict) + and admin_metadata.get(metadata_key) is True + ): + return True + return False + + +def _key_or_team_allows_client_mock_response( + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + return _key_or_team_metadata_flag_is_true( + user_api_key_dict=user_api_key_dict, + metadata_key=_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY, + ) + + +def _key_or_team_allows_client_message_redaction_opt_out( + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + return _key_or_team_metadata_flag_is_true( + user_api_key_dict=user_api_key_dict, + metadata_key=_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY, + ) + def _get_metadata_variable_name(request: Request) -> str: """ @@ -228,13 +335,25 @@ def convert_key_logging_metadata_to_callback( for var, value in data.callback_vars.items(): if team_callback_settings_obj.callback_vars is None: team_callback_settings_obj.callback_vars = {} - team_callback_settings_obj.callback_vars[var] = str( - litellm.utils.get_secret(value, default_value=value) or value - ) + team_callback_settings_obj.callback_vars[var] = str(value) return team_callback_settings_obj +def _get_validated_callback_metadata( + item: dict, *, source: str +) -> Optional[AddTeamCallback]: + try: + return AddTeamCallback(**item) + except (PydanticValidationError, ValueError) as e: + verbose_proxy_logger.warning( + "Ignoring invalid %s callback metadata: %s", + source, + _sanitize_for_log(str(e)), + ) + return None + + class KeyAndTeamLoggingSettings: """ Helper class to get the dynamic logging settings for the key and team @@ -274,8 +393,11 @@ def _get_dynamic_logging_metadata( ######################################################################################### if key_dynamic_logging_settings is not None: for item in key_dynamic_logging_settings: + callback = _get_validated_callback_metadata(item=item, source="key-level") + if callback is None: + continue callback_settings_obj = convert_key_logging_metadata_to_callback( - data=AddTeamCallback(**item), + data=callback, team_callback_settings_obj=callback_settings_obj, ) ######################################################################################### @@ -283,8 +405,11 @@ def _get_dynamic_logging_metadata( ######################################################################################### elif team_dynamic_logging_settings is not None: for item in team_dynamic_logging_settings: + callback = _get_validated_callback_metadata(item=item, source="team-level") + if callback is None: + continue callback_settings_obj = convert_key_logging_metadata_to_callback( - data=AddTeamCallback(**item), + data=callback, team_callback_settings_obj=callback_settings_obj, ) ######################################################################################### @@ -904,6 +1029,14 @@ class LiteLLMProxyRequestSetup: callback_vars_dict.pop("team_id", None) callback_vars_dict.pop("success_callback", None) callback_vars_dict.pop("failure_callback", None) + callback_vars_dict = { + key: ( + litellm.utils.get_secret(value, default_value=value) or value + if isinstance(value, str) + else value + ) + for key, value in callback_vars_dict.items() + } return TeamCallbackMetadata( success_callback=team_config.get("success_callback", None), @@ -962,11 +1095,15 @@ async def add_litellm_data_to_request( # noqa: PLR0915 # Strip internal-only keys from user input before the proxy sets its own. # These keys are injected by the proxy itself below — user-supplied values # must not be trusted. - for _internal_key in ( - "proxy_server_request", - "standard_logging_object", - "secret_fields", - ): + _allow_client_mock_response = _key_or_team_allows_client_mock_response( + user_api_key_dict + ) + _allow_client_message_redaction_opt_out = ( + _key_or_team_allows_client_message_redaction_opt_out(user_api_key_dict) + ) + for _internal_key in _UNTRUSTED_ROOT_CONTROL_FIELDS: + if _allow_client_mock_response and _internal_key in _CLIENT_MOCK_CONTROL_FIELDS: + continue data.pop(_internal_key, None) # Strip spoofable auth metadata from user-supplied metadata dict _user_metadata = data.get("metadata") @@ -1007,6 +1144,17 @@ async def add_litellm_data_to_request( # noqa: PLR0915 forward_llm_provider_auth_headers=forward_llm_auth, authenticated_with_header=authenticated_with_header, ) + _strip_untrusted_request_header_controls( + _headers, + allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, + ) + if ( + not _allow_client_message_redaction_opt_out + and litellm.turn_off_message_logging is True + and "turn_off_message_logging" in data + and _is_false_like(data["turn_off_message_logging"]) + ): + data.pop("turn_off_message_logging", None) verbose_proxy_logger.debug(f"Request Headers: {_headers}") verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") @@ -1144,8 +1292,18 @@ async def add_litellm_data_to_request( # noqa: PLR0915 for _meta_key in ("metadata", "litellm_metadata"): _user_meta = data.get(_meta_key) if isinstance(_user_meta, dict): - _user_meta.pop("_pipeline_managed_guardrails", None) - for _k in [k for k in _user_meta if k.startswith("user_api_key_")]: + _strip_untrusted_request_header_controls( + _user_meta.get("headers"), + allow_client_message_redaction_opt_out=( + _allow_client_message_redaction_opt_out + ), + ) + for _k in [ + k + for k in _user_meta + if k.startswith("user_api_key_") + or k in _UNTRUSTED_METADATA_CONTROL_FIELDS + ]: _user_meta.pop(_k, None) # Strip caller-supplied routing/budget tags unless the admin has opted diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c6d37ace4f..921d24da04 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2069,6 +2069,9 @@ async def delete_user( litellm_proxy_admin_name, prisma_client, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -2162,9 +2165,11 @@ async def delete_user( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.USER_TABLE_NAME, object_id=user_id, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a424f1558f..2485aea14f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._experimental.mcp_server.db import ( rotate_mcp_server_credentials_master_key, + rotate_mcp_user_credentials_master_key, ) from litellm.proxy._types import * from litellm.proxy._types import LiteLLM_VerificationToken @@ -65,6 +66,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( attach_object_permission_to_dict, handle_update_object_permission_common, validate_key_mcp_servers_against_team, + validate_key_search_tools_against_team, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -768,6 +770,10 @@ async def _common_key_generation_helper( # noqa: PLR0915 object_permission=data_json.get("object_permission"), team_obj=team_table, ) + await validate_key_search_tools_against_team( + object_permission=data_json.get("object_permission"), + team_obj=team_table, + ) data_json = await _set_object_permission( data_json=data_json, @@ -2010,6 +2016,10 @@ async def _validate_mcp_servers_for_key_update( object_permission=object_permission_dict, team_obj=effective_team_obj, ) + await validate_key_search_tools_against_team( + object_permission=object_permission_dict, + team_obj=effective_team_obj, + ) async def _validate_update_key_data( @@ -3709,6 +3719,17 @@ async def _rotate_master_key( # noqa: PLR0915 "Failed to rotate MCP server credentials: %s", str(e) ) + # 4b. process MCP user-scoped credentials table (BYOK + OAuth2 tokens) + try: + await rotate_mcp_user_credentials_master_key( + prisma_client=prisma_client, + new_master_key=new_master_key, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to rotate MCP user credentials: %s", str(e) + ) + # 5. process credentials table try: credentials = await prisma_client.db.litellm_credentialstable.find_many() @@ -5245,6 +5266,9 @@ async def block_key( proxy_logging_obj, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value)) @@ -5288,9 +5312,11 @@ async def block_key( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, @@ -5354,6 +5380,9 @@ async def unblock_key( proxy_logging_obj, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value)) @@ -5397,9 +5426,11 @@ async def unblock_key( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, @@ -5580,7 +5611,6 @@ async def test_key_logging( "content": "Hello, this is a test from litellm /key/health. No LLM API call was made for this", } ], - "mock_response": "test response", } data = await add_litellm_data_to_request( data=data, @@ -5589,6 +5619,7 @@ async def test_key_logging( general_settings=general_settings, request=request, ) + data["mock_response"] = "test response" await litellm.acompletion( **data ) # make mock completion call to trigger key based callbacks diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index fca08e591f..9c510a568e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -56,6 +56,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.management_helpers.audit_logs import get_audit_log_changed_by router = APIRouter(prefix="/v1/mcp", tags=["mcp"]) @@ -2230,7 +2231,12 @@ if MCP_AVAILABLE: detail={"error": "Only proxy admins can create MCP toolsets."}, ) touched_by = ( - litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME + get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME, + ) + or LITELLM_PROXY_ADMIN_NAME ) try: result = await create_mcp_toolset(prisma_client, payload, touched_by) @@ -2321,7 +2327,12 @@ if MCP_AVAILABLE: detail={"error": "Only proxy admins can update MCP toolsets."}, ) touched_by = ( - litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME + get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME, + ) + or LITELLM_PROXY_ADMIN_NAME ) try: result = await update_mcp_toolset(prisma_client, payload, touched_by) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 4eec7c6b7c..a6c0c7dcc0 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -4,15 +4,22 @@ Endpoints to control callbacks per team Use this when each team should control its own callbacks """ +import asyncio +import copy import json import traceback -from typing import List, Optional +from datetime import datetime, timezone +from typing import Any, List, Optional from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +import litellm from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.proxy._types import ( AddTeamCallback, + LiteLLM_AuditLogs, + LitellmTableNames, ProxyErrorTypes, ProxyException, TeamCallbackMetadata, @@ -24,6 +31,106 @@ from litellm.proxy.management_helpers.utils import management_endpoint_wrapper router = APIRouter() +_CALLBACK_VARS_REDACTED = "***REDACTED***" + + +def _redact_callback_secrets(metadata: Any) -> Any: + """Strip secret values out of a team-metadata snapshot before audit logging. + + Both ``team_metadata["logging"]`` (list of ``AddTeamCallback`` dicts) and + ``team_metadata["callback_settings"]["callback_vars"]`` carry provider + credentials such as ``langfuse_secret_key``, ``langsmith_api_key``, and + ``gcs_path_service_account``. Persisting them verbatim into + ``LiteLLM_AuditLogs`` would let anyone with read access to the audit + table harvest team callback credentials, so we replace each value with + a fixed marker. The keys themselves are kept so the audit reader can + still see *which* fields changed. + """ + if not isinstance(metadata, dict): + return metadata + redacted = copy.deepcopy(metadata) + logging_entries = redacted.get("logging") + if isinstance(logging_entries, list): + for entry in logging_entries: + if isinstance(entry, dict) and isinstance(entry.get("callback_vars"), dict): + entry["callback_vars"] = { + k: _CALLBACK_VARS_REDACTED for k in entry["callback_vars"] + } + callback_settings = redacted.get("callback_settings") + if isinstance(callback_settings, dict) and isinstance( + callback_settings.get("callback_vars"), dict + ): + callback_settings["callback_vars"] = { + k: _CALLBACK_VARS_REDACTED for k in callback_settings["callback_vars"] + } + return redacted + + +def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: + """Surface a fire-and-forget audit-log task failure. + + ``asyncio.create_task`` swallows exceptions silently — if the audit + write fails (transient DB error etc.) we'd otherwise lose the row + without any signal. Log at warning level so the operator sees there's + a gap in the audit trail. + """ + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + verbose_proxy_logger.warning("Failed to write team-callback audit log: %s", exc) + + +async def _emit_team_callback_audit_log( + *, + team_id: str, + before_metadata: Any, + after_metadata: Any, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: Optional[str], +) -> None: + """Emit an audit-log row for a team-callback mutation. + + Mirrors the ``store_audit_logs``-gated pattern used in + ``team_endpoints.py``: the call is async-fire-and-forget and is a no-op + when audit logging is not enabled on the proxy. Captured under + ``LitellmTableNames.TEAM_TABLE_NAME`` so the row co-locates with other + team mutations in the audit table. + + Callback secrets are redacted before serialization so the audit table + cannot itself become a credential-harvest sink. + """ + if litellm.store_audit_logs is not True: + return + + from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + ) + from litellm.proxy.proxy_server import litellm_proxy_admin_name + + redacted_before = _redact_callback_secrets(before_metadata) + redacted_after = _redact_callback_secrets(after_metadata) + + task = asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=litellm_changed_by + or user_api_key_dict.user_id + or litellm_proxy_admin_name, + changed_by_api_key=user_api_key_dict.api_key, + table_name=LitellmTableNames.TEAM_TABLE_NAME, + object_id=team_id, + action="updated", + updated_values=json.dumps({"metadata": redacted_after}, default=str), + before_value=json.dumps({"metadata": redacted_before}, default=str), + ) + ) + ) + task.add_done_callback(_log_audit_task_exception) + + @router.post( "/team/{team_id:path}/callback", tags=["team management"], @@ -123,6 +230,7 @@ async def add_team_callbacks( param="callback_name", ) + before_metadata = copy.deepcopy(team_metadata) team_callback_settings.append(data.model_dump()) team_metadata["logging"] = team_callback_settings @@ -132,6 +240,14 @@ async def add_team_callbacks( where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore ) + await _emit_team_callback_audit_log( + team_id=team_id, + before_metadata=before_metadata, + after_metadata=team_metadata, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + return { "status": "success", "data": new_team_row, @@ -165,6 +281,10 @@ async def disable_team_logging( http_request: Request, team_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), ): """ Disable all logging callbacks for a team @@ -198,6 +318,7 @@ async def disable_team_logging( # Update team metadata to disable logging team_metadata = _existing_team.metadata + before_metadata = copy.deepcopy(team_metadata) team_callback_settings = team_metadata.get("callback_settings", {}) team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) @@ -222,6 +343,17 @@ async def disable_team_logging( }, ) + # Disabling a team's logging callbacks is itself a logging-control + # action — emit an audit-log row so the action remains traceable + # even though the team's own observability is now off. + await _emit_team_callback_audit_log( + team_id=team_id, + before_metadata=before_metadata, + after_metadata=team_metadata, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + return { "status": "success", "message": f"Logging disabled for team {team_id}", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f254fea3e7..61247ce5de 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -906,6 +906,9 @@ async def new_team( # noqa: PLR0915 prisma_client, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -1174,9 +1177,11 @@ async def new_team( # noqa: PLR0915 request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=data.team_id, @@ -1214,7 +1219,10 @@ async def _create_team_update_audit_log( user_api_key_dict: User API key authentication details litellm_proxy_admin_name: Name of the proxy admin """ - from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update + from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + get_audit_log_changed_by, + ) _before_value = existing_team_row.json(exclude_none=True) _before_value = json.dumps(_before_value, default=str) @@ -1225,9 +1233,11 @@ async def _create_team_update_audit_log( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=team_id, @@ -2003,21 +2013,34 @@ def team_member_add_duplication_check( async def _validate_team_member_add_permissions( user_api_key_dict: UserAPIKeyAuth, complete_team_data: LiteLLM_TeamTable, + data: TeamMemberAddRequest, ) -> None: - """Validate if user has permission to add members to the team.""" + """Validate if user has permission to add members to the team. + + Standard users can self-join an *available team*, but the bypass + must not be allowed to escalate them to ``role=admin`` or to add + other users into the team. When access is granted via the + available-team bypass we therefore enforce that every member in + the request matches the caller's own ``user_id`` and is being + added with ``role="user"``. + """ if ( - hasattr(user_api_key_dict, "user_role") - and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not _is_user_team_admin( - user_api_key_dict=user_api_key_dict, team_obj=complete_team_data - ) - and not await _is_user_org_admin_for_team( - user_api_key_dict=user_api_key_dict, team_obj=complete_team_data - ) - and not _is_available_team( - team_id=complete_team_data.team_id, - user_api_key_dict=user_api_key_dict, - ) + getattr(user_api_key_dict, "user_role", None) + == LitellmUserRoles.PROXY_ADMIN.value + ): + return + if _is_user_team_admin( + user_api_key_dict=user_api_key_dict, team_obj=complete_team_data + ): + return + if await _is_user_org_admin_for_team( + user_api_key_dict=user_api_key_dict, team_obj=complete_team_data + ): + return + + if not _is_available_team( + team_id=complete_team_data.team_id, + user_api_key_dict=user_api_key_dict, ): raise HTTPException( status_code=403, @@ -2029,6 +2052,34 @@ async def _validate_team_member_add_permissions( }, ) + # Available-team self-join: caller may add only themselves, only as a + # standard user. Enforce that here so the bypass cannot be used as a + # privilege-escalation or cross-user-injection primitive. + members = data.member if isinstance(data.member, list) else [data.member] + caller_user_id = getattr(user_api_key_dict, "user_id", None) + for member in members: + if getattr(member, "role", "user") != "user": + raise HTTPException( + status_code=403, + detail={ + "error": ( + "Available-team self-join cannot assign 'admin' role. " + "Only proxy/team/org admins can add admins to a team." + ) + }, + ) + member_user_id = getattr(member, "user_id", None) + if not caller_user_id or not member_user_id or member_user_id != caller_user_id: + raise HTTPException( + status_code=403, + detail={ + "error": ( + "Available-team self-join can only add the caller " + "(user_id must match the authenticated user's user_id)." + ) + }, + ) + async def _process_team_members( data: TeamMemberAddRequest, @@ -2049,8 +2100,11 @@ async def _process_team_members( # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models member_allowed_models = data.allowed_models - if member_allowed_models is None and complete_team_data.default_team_member_models: - member_allowed_models = complete_team_data.default_team_member_models + team_default_member_models = getattr( + complete_team_data, "default_team_member_models", None + ) + if member_allowed_models is None and team_default_member_models: + member_allowed_models = team_default_member_models if isinstance(data.member, Member): try: @@ -2381,6 +2435,7 @@ async def team_member_add( await _validate_team_member_add_permissions( user_api_key_dict=user_api_key_dict, complete_team_data=complete_team_data, + data=data, ) # Validate and populate user_email/user_id for members before processing @@ -2992,6 +3047,9 @@ async def delete_team( litellm_proxy_admin_name, prisma_client, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -3051,9 +3109,11 @@ async def delete_team( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=team_id, @@ -4694,6 +4754,8 @@ async def update_team_member_permissions( complete_team_data = LiteLLM_TeamTable(**existing_team_row.model_dump()) + # Available-team self-join must NOT grant write access to team-wide + # permission policies; only proxy/team/org admins can update them. if ( hasattr(user_api_key_dict, "user_role") and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value @@ -4703,16 +4765,12 @@ async def update_team_member_permissions( and not await _is_user_org_admin_for_team( user_api_key_dict=user_api_key_dict, team_obj=complete_team_data ) - and not _is_available_team( - team_id=complete_team_data.team_id, - user_api_key_dict=user_api_key_dict, - ) ): raise HTTPException( status_code=403, detail={ "error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format( - "/team/member_add", + "/team/permissions_update", complete_team_data.team_id, ) }, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 46e7963da7..c4564a4eb0 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -13,7 +13,9 @@ import base64 import hashlib import inspect import os +import re import secrets +from html import escape from copy import deepcopy from typing import ( TYPE_CHECKING, @@ -27,13 +29,13 @@ from typing import ( Union, cast, ) -from urllib.parse import urlencode, urlparse +from urllib.parse import parse_qs, urlencode, urlparse if TYPE_CHECKING: import httpx import jwt -from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from fastapi.responses import RedirectResponse import litellm @@ -41,6 +43,9 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.caching import DualCache from litellm.constants import ( + CLI_SSO_SESSION_CACHE_KEY_PREFIX, + CLI_SSO_SESSION_TTL_SECONDS, + LITELLM_CLI_SOURCE_IDENTIFIER, LITELLM_UI_SESSION_DURATION, MAX_SPENDLOG_ROWS_TO_QUERY, MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE, @@ -70,7 +75,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_user_object -from litellm.proxy.auth.auth_utils import _has_user_setup_sso +from litellm.proxy.auth.auth_utils import _get_request_ip_address, _has_user_setup_sso from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.admin_ui_utils import ( @@ -123,6 +128,250 @@ router = APIRouter() # Metadata fields (token_type, expires_in, scope) are intentionally kept so # response convertors see the same fields in the PKCE path as in the non-PKCE path. _OAUTH_TOKEN_FIELDS = frozenset({"access_token", "id_token", "refresh_token"}) +_CLI_SSO_FLOW_CACHE_KEY_PREFIX = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:flow" +_CLI_SSO_START_RATE_LIMIT_CACHE_KEY_PREFIX = ( + f"{_CLI_SSO_FLOW_CACHE_KEY_PREFIX}:start_rate_limit" +) +_CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS = 60 +_CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS = 30 +_CLI_SSO_USER_CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" +_CLI_SSO_LOGIN_ID_RE = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") + + +def _hash_cli_sso_secret(secret: str) -> str: + return hashlib.sha256(secret.encode("utf-8")).hexdigest() + + +def _normalize_cli_sso_user_code(user_code: str) -> str: + return "".join(ch for ch in user_code.upper() if ch.isalnum()) + + +def _generate_cli_sso_user_code() -> str: + user_code = "".join(secrets.choice(_CLI_SSO_USER_CODE_ALPHABET) for _ in range(8)) + return f"{user_code[:4]}-{user_code[4:]}" + + +def _get_cli_sso_flow_cache_key(login_id: str) -> str: + return f"{_CLI_SSO_FLOW_CACHE_KEY_PREFIX}:{login_id}" + + +def _is_valid_cli_sso_login_id(login_id: Optional[str]) -> bool: + return isinstance(login_id, str) and bool(_CLI_SSO_LOGIN_ID_RE.fullmatch(login_id)) + + +def _get_cli_sso_start_rate_limit_cache_key( + request: Request, use_x_forwarded_for: Optional[bool] = False +) -> str: + client_ip = ( + _get_request_ip_address( + request=request, use_x_forwarded_for=use_x_forwarded_for + ) + or "unknown" + ) + client_ip_hash = _hash_cli_sso_secret(client_ip) + return f"{_CLI_SSO_START_RATE_LIMIT_CACHE_KEY_PREFIX}:{client_ip_hash}" + + +def _check_cli_sso_start_rate_limit( + request: Request, + cache: DualCache, + use_x_forwarded_for: Optional[bool] = False, +) -> None: + rate_limit_cache_key = _get_cli_sso_start_rate_limit_cache_key( + request=request, use_x_forwarded_for=use_x_forwarded_for + ) + current_attempts = cache.increment_cache( + key=rate_limit_cache_key, + value=1, + ttl=_CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS, + ) + if current_attempts > _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: + raise HTTPException( + status_code=429, + detail="Too many CLI login attempts. Try again later.", + ) + + +def _get_cli_sso_flow_or_raise(login_id: Optional[str], cache: DualCache) -> dict: + if not _is_valid_cli_sso_login_id(login_id): + raise HTTPException(status_code=400, detail="Invalid CLI login session") + + cache_key = _get_cli_sso_flow_cache_key(cast(str, login_id)) + flow = cache.get_cache(key=cache_key) + if not isinstance(flow, dict) or "poll_secret_hash" not in flow: + raise HTTPException(status_code=400, detail="Invalid CLI login session") + return flow + + +def _set_cli_sso_flow(login_id: str, cache: DualCache, flow: dict) -> None: + cache.set_cache( + key=_get_cli_sso_flow_cache_key(login_id), + value=flow, + ttl=CLI_SSO_SESSION_TTL_SECONDS, + ) + + +def _verify_cli_sso_poll_secret(flow: dict, poll_secret: Optional[str]) -> bool: + expected_poll_secret_hash = flow.get("poll_secret_hash") + if not isinstance(expected_poll_secret_hash, str) or not isinstance( + poll_secret, str + ): + return False + supplied_poll_secret_hash = _hash_cli_sso_secret(poll_secret) + return secrets.compare_digest(supplied_poll_secret_hash, expected_poll_secret_hash) + + +def _render_cli_sso_verification_page( + verify_url: str, browser_complete_token: str +) -> str: + escaped_verify_url = escape(verify_url, quote=True) + escaped_browser_complete_token = escape(browser_complete_token, quote=True) + return f""" + + + + LiteLLM CLI Login + + + +
+

Complete CLI Login

+

Enter the verification code shown in your terminal to finish this login.

+
+ + + + + +
+ + + """ + + +@router.post("/sso/cli/start", tags=["experimental"], include_in_schema=False) +async def cli_sso_start(request: Request): + from litellm.proxy.proxy_server import general_settings, user_api_key_cache + + _check_cli_sso_start_rate_limit( + request=request, + cache=user_api_key_cache, + use_x_forwarded_for=bool( + (general_settings or {}).get("use_x_forwarded_for", False) + ), + ) + + login_id = f"cli-{secrets.token_urlsafe(24)}" + poll_secret = secrets.token_urlsafe(32) + user_code = _generate_cli_sso_user_code() + + flow = { + "poll_secret_hash": _hash_cli_sso_secret(poll_secret), + "user_code_hash": _hash_cli_sso_secret(_normalize_cli_sso_user_code(user_code)), + "sso_complete": False, + "user_code_verified": False, + "session_data": None, + } + _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow) + + return { + "login_id": login_id, + "poll_secret": poll_secret, + "user_code": user_code, + "expires_in": CLI_SSO_SESSION_TTL_SECONDS, + } + + +@router.post( + "/sso/cli/complete/{login_id}", tags=["experimental"], include_in_schema=False +) +async def cli_sso_complete(request: Request, login_id: str): + from fastapi.responses import HTMLResponse + + from litellm.proxy.common_utils.html_forms.cli_sso_success import ( + render_cli_sso_success_page, + ) + from litellm.proxy.proxy_server import user_api_key_cache + + flow = _get_cli_sso_flow_or_raise(login_id=login_id, cache=user_api_key_cache) + if not flow.get("sso_complete") or not flow.get("session_data"): + raise HTTPException(status_code=400, detail="CLI login is not ready") + + body = (await request.body()).decode("utf-8") + form_values = parse_qs(body) + supplied_user_code = (form_values.get("user_code") or [""])[0] + supplied_browser_complete_token = ( + form_values.get("browser_complete_token") or [""] + )[0] + supplied_user_code_hash = _hash_cli_sso_secret( + _normalize_cli_sso_user_code(supplied_user_code) + ) + supplied_browser_complete_token_hash = _hash_cli_sso_secret( + supplied_browser_complete_token + ) + + expected_user_code_hash = flow.get("user_code_hash") + if not isinstance(expected_user_code_hash, str) or not secrets.compare_digest( + supplied_user_code_hash, expected_user_code_hash + ): + raise HTTPException(status_code=400, detail="Invalid verification code") + + expected_browser_complete_token_hash = flow.get("browser_complete_token_hash") + if not isinstance( + expected_browser_complete_token_hash, str + ) or not secrets.compare_digest( + supplied_browser_complete_token_hash, expected_browser_complete_token_hash + ): + raise HTTPException(status_code=400, detail="Invalid verification code") + + flow["user_code_verified"] = True + _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow) + + html_content = render_cli_sso_success_page() + return HTMLResponse(content=html_content, status_code=200) def normalize_email(email: Optional[str]) -> Optional[str]: @@ -333,6 +582,7 @@ async def google_login( from litellm.proxy.proxy_server import ( premium_user, prisma_client, + user_api_key_cache, user_custom_ui_sso_sign_in_handler, ) @@ -382,14 +632,15 @@ async def google_login( redirect_url = SSOAuthenticationHandler.get_redirect_url_for_sso( request=request, sso_callback_route="sso/callback", - existing_key=existing_key, ) - # Store CLI key in state for OAuth flow + if source == LITELLM_CLI_SOURCE_IDENTIFIER: + _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache) + + # Store CLI login handle in state for OAuth flow cli_state: Optional[str] = SSOAuthenticationHandler._get_cli_state( source=source, key=key, - existing_key=existing_key, ) # check if user defined a custom auth sso sign in handler, if yes, use it @@ -1392,18 +1643,12 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) if state and state.startswith(f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:"): - # Extract the key ID and existing_key from the state - # State format: {PREFIX}:{key}:{existing_key} or {PREFIX}:{key} - state_parts = state.split(":", 2) # Split into max 3 parts + # State format: {PREFIX}:{login_id} + state_parts = state.split(":", 1) key_id = state_parts[1] if len(state_parts) > 1 else None - existing_key = state_parts[2] if len(state_parts) > 2 else None - verbose_proxy_logger.info( - f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}" - ) - return await cli_sso_callback( - request=request, key=key_id, existing_key=existing_key, result=result - ) + verbose_proxy_logger.info("CLI SSO callback detected") + return await cli_sso_callback(request=request, key=key_id, result=result) # Control-plane cross-origin: read return_to from cookie. # Starlette's cookie_parser already handles RFC 2109 unquoting. @@ -1424,13 +1669,10 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: async def cli_sso_callback( request: Request, key: Optional[str] = None, - existing_key: Optional[str] = None, result: Optional[Union[OpenID, dict]] = None, ): """CLI SSO callback - stores session info for JWT generation on polling""" - verbose_proxy_logger.info( - f"CLI SSO callback for key: {key}, existing_key: {existing_key}" - ) + verbose_proxy_logger.info("CLI SSO callback") from litellm.proxy.proxy_server import ( prisma_client, @@ -1438,11 +1680,7 @@ async def cli_sso_callback( user_api_key_cache, ) - if not key or not key.startswith("sk-"): - raise HTTPException( - status_code=400, - detail="Invalid key parameter. Must be a valid key ID starting with 'sk-'", - ) + flow = _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache) if prisma_client is None: raise HTTPException( @@ -1480,9 +1718,6 @@ async def cli_sso_callback( status_code=500, detail="Failed to retrieve user information from SSO" ) - # Store session info in cache (10 min TTL) - from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX - # Get all teams from user_info - CLI will let user select which one teams: List[str] = [] if hasattr(user_info, "teams") and user_info.teams: @@ -1523,21 +1758,25 @@ async def cli_sso_callback( "team_details": team_details, } - cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key}" - user_api_key_cache.set_cache(key=cache_key, value=session_data, ttl=600) + flow["session_data"] = session_data + flow["sso_complete"] = True + browser_complete_token = secrets.token_urlsafe(32) + flow["browser_complete_token_hash"] = _hash_cli_sso_secret( + browser_complete_token + ) + _set_cli_sso_flow(login_id=cast(str, key), cache=user_api_key_cache, flow=flow) verbose_proxy_logger.info( f"Stored CLI SSO session for user: {user_info.user_id}, teams: {teams}, num_teams: {len(teams)}" ) - # Return success page from fastapi.responses import HTMLResponse - from litellm.proxy.common_utils.html_forms.cli_sso_success import ( - render_cli_sso_success_page, + verify_url = str(request.url_for("cli_sso_complete", login_id=key)) + html_content = _render_cli_sso_verification_page( + verify_url=verify_url, + browser_complete_token=browser_complete_token, ) - - html_content = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) except Exception as e: @@ -1548,7 +1787,11 @@ async def cli_sso_callback( @router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False) -async def cli_poll_key(key_id: str, team_id: Optional[str] = None): +async def cli_poll_key( + key_id: str, + team_id: Optional[str] = None, + x_litellm_cli_poll_secret: Optional[str] = Header(default=None), +): """ CLI polling endpoint - retrieves session from cache and generates JWT. @@ -1557,22 +1800,25 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): 2. Second poll (with team_id): Generates JWT with selected team and deletes session Args: - key_id: The session key ID + key_id: The CLI login session ID team_id: Optional team ID to assign to the JWT. If provided, must be one of user's teams. """ - from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken from litellm.proxy.proxy_server import user_api_key_cache - if not key_id.startswith("sk-"): - raise HTTPException(status_code=400, detail="Invalid key ID format") - try: - # Look up session in cache - cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key_id}" - session_data = user_api_key_cache.get_cache(key=cache_key) + flow = _get_cli_sso_flow_or_raise(login_id=key_id, cache=user_api_key_cache) + if not _verify_cli_sso_poll_secret( + flow=flow, poll_secret=x_litellm_cli_poll_secret + ): + raise HTTPException(status_code=403, detail="Invalid CLI polling secret") - if session_data: + if not flow.get("sso_complete") or not flow.get("user_code_verified"): + return {"status": "pending"} + + session_data = flow.get("session_data") + + if isinstance(session_data, dict): user_teams = session_data.get("teams", []) user_team_details = session_data.get("team_details") user_id = session_data["user_id"] @@ -1632,7 +1878,7 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): ) # Delete cache entry (single-use) - user_api_key_cache.delete_cache(key=cache_key) + user_api_key_cache.delete_cache(key=_get_cli_sso_flow_cache_key(key_id)) verbose_proxy_logger.info( f"CLI JWT generated for user: {user_id}, team: {team_id}" @@ -1650,6 +1896,8 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): else: return {"status": "pending"} + except HTTPException: + raise except Exception as e: verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}") raise HTTPException( @@ -2393,20 +2641,15 @@ class SSOAuthenticationHandler: This is used to authenticate through the CLI login flow. - The state parameter format is: {PREFIX}:{key}:{existing_key} - - If existing_key is provided, it's included in the state + The state parameter format is: {PREFIX}:{login_id} - The state parameter is used to pass data through the OAuth flow without changing the callback URL """ from litellm.constants import ( LITELLM_CLI_SESSION_TOKEN_PREFIX, - LITELLM_CLI_SOURCE_IDENTIFIER, ) if source == LITELLM_CLI_SOURCE_IDENTIFIER and key: - if existing_key: - return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}:{existing_key}" - else: - return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}" + return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}" else: return None diff --git a/litellm/proxy/management_endpoints/workflow_management_endpoints.py b/litellm/proxy/management_endpoints/workflow_management_endpoints.py new file mode 100644 index 0000000000..a19af4dd48 --- /dev/null +++ b/litellm/proxy/management_endpoints/workflow_management_endpoints.py @@ -0,0 +1,492 @@ +""" +WORKFLOW RUN MANAGEMENT + +Generic durable state tracking for agents and automated workflows. + +POST /v1/workflows/runs - Create a workflow run +GET /v1/workflows/runs - List runs (filter by type, status) +GET /v1/workflows/runs/{run_id} - Get run with latest event +PATCH /v1/workflows/runs/{run_id} - Update status, metadata, output +POST /v1/workflows/runs/{run_id}/events - Append event (updates run status) +GET /v1/workflows/runs/{run_id}/events - Full event log +POST /v1/workflows/runs/{run_id}/messages - Append conversation message +GET /v1/workflows/runs/{run_id}/messages - Fetch conversation history +""" + +import json +from typing import Any, Dict, Literal, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +try: + from prisma.errors import UniqueViolationError +except ImportError: + UniqueViolationError = None # type: ignore +from pydantic import BaseModel + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + +_MAX_SEQUENCE_RETRIES = 5 + + +def _json(value: Any) -> str: + """Serialize a Python value for prisma-client-py Json fields (must be a string).""" + return json.dumps(value) + + +def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: + return user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + + +def _caller_key(user_api_key_dict: UserAPIKeyAuth) -> Optional[str]: + """Return the hashed key token that identifies this caller, or None for master key.""" + return user_api_key_dict.token + + +# Status transitions driven by event_type +_EVENT_STATUS_MAP: Dict[str, str] = { + "step.started": "running", + "step.failed": "failed", + "hook.waiting": "paused", + "hook.received": "running", +} + + +# --------------------------------------------------------------------------- +# Request / Response models +# --------------------------------------------------------------------------- + + +class WorkflowRunCreateRequest(BaseModel): + workflow_type: str + input: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + + +WorkflowRunStatus = Literal["pending", "running", "paused", "completed", "failed"] + + +class WorkflowRunUpdateRequest(BaseModel): + status: Optional[WorkflowRunStatus] = None + output: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + + +class WorkflowEventCreateRequest(BaseModel): + event_type: str + step_name: str + data: Optional[Dict[str, Any]] = None + + +class WorkflowMessageCreateRequest(BaseModel): + role: str + content: str + session_id: Optional[str] = None + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def _get_next_sequence_number(prisma_client: Any, run_id: str, table: str) -> int: + """Return MAX(sequence_number) + 1 for the given run, for either events or messages.""" + if table == "events": + rows = await prisma_client.db.litellm_workflowevent.find_many( + where={"run_id": run_id}, + order={"sequence_number": "desc"}, + take=1, + ) + else: + rows = await prisma_client.db.litellm_workflowmessage.find_many( + where={"run_id": run_id}, + order={"sequence_number": "desc"}, + take=1, + ) + return (rows[0].sequence_number + 1) if rows else 0 + + +async def _require_run( + prisma_client: Any, + run_id: str, + user_api_key_dict: Optional[UserAPIKeyAuth] = None, +) -> Any: + """Return the run or raise 404. For non-admin callers, also enforce key ownership.""" + run = await prisma_client.db.litellm_workflowrun.find_unique( + where={"run_id": run_id} + ) + if run is None: + raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") + if user_api_key_dict is not None and not _is_admin(user_api_key_dict): + caller = _caller_key(user_api_key_dict) + if not caller or run.created_by != caller: + raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") + return run + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.post( + "/v1/workflows/runs", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def create_workflow_run( + data: WorkflowRunCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Create a new workflow run. Returns run_id and session_id. + + The caller's API key token is stored as created_by so that non-admin keys + can only see and modify their own runs. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + try: + create_data: Dict[str, Any] = { + "workflow_type": data.workflow_type, + "created_by": _caller_key(user_api_key_dict), + } + if data.input is not None: + create_data["input"] = _json(data.input) + if data.metadata is not None: + create_data["metadata"] = _json(data.metadata) + run = await prisma_client.db.litellm_workflowrun.create(data=create_data) + return run + except Exception as e: + verbose_proxy_logger.exception("Error creating workflow run: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/v1/workflows/runs", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_workflow_runs( + workflow_type: Optional[str] = Query(None), + status: Optional[str] = Query(None), + limit: int = Query(50, ge=1, le=250), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """List workflow runs. Filter by workflow_type and/or status. + + Non-admin callers only see runs created by their own API key. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + where: Dict[str, Any] = {} + if workflow_type: + where["workflow_type"] = workflow_type + if status: + statuses = [s.strip() for s in status.split(",")] + where["status"] = {"in": statuses} if len(statuses) > 1 else statuses[0] + + # Non-admin callers are scoped to their own key. + if not _is_admin(user_api_key_dict): + caller = _caller_key(user_api_key_dict) + if caller: + where["created_by"] = caller + + try: + runs = await prisma_client.db.litellm_workflowrun.find_many( + where=where, + order={"created_at": "desc"}, + take=limit, + ) + return {"runs": runs, "count": len(runs)} + except Exception as e: + verbose_proxy_logger.exception("Error listing workflow runs: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/v1/workflows/runs/{run_id}", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_workflow_run( + run_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Get a workflow run with its most recent event.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + try: + run = await prisma_client.db.litellm_workflowrun.find_unique( + where={"run_id": run_id}, + include={"events": {"order_by": {"sequence_number": "desc"}, "take": 1}}, + ) + if run is None: + raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") + if not _is_admin(user_api_key_dict): + caller = _caller_key(user_api_key_dict) + if not caller or run.created_by != caller: + raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") + return run + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception("Error getting workflow run: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch( + "/v1/workflows/runs/{run_id}", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_workflow_run( + run_id: str, + data: WorkflowRunUpdateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Update status, metadata, or output on a workflow run.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + update: Dict[str, Any] = {} + if data.status is not None: + update["status"] = data.status + if data.output is not None: + update["output"] = _json(data.output) + if data.metadata is not None: + update["metadata"] = _json(data.metadata) + + if not update: + raise HTTPException(status_code=400, detail="No fields to update") + + # Enforce ownership before writing. + await _require_run(prisma_client, run_id, user_api_key_dict) + + try: + run = await prisma_client.db.litellm_workflowrun.update( + where={"run_id": run_id}, + data=update, + ) + if run is None: + raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") + return run + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception("Error updating workflow run: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/v1/workflows/runs/{run_id}/events", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def append_workflow_event( + run_id: str, + data: WorkflowEventCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Append an event to the run's event log. Also updates run.status if event_type maps to a status. + + Sequence numbers use optimistic concurrency: on a unique-constraint collision + (concurrent append), retries up to _MAX_SEQUENCE_RETRIES times with a fresh MAX+1. + The event+status update is atomic in a single DB transaction. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + await _require_run(prisma_client, run_id, user_api_key_dict) + + new_status = _EVENT_STATUS_MAP.get(data.event_type) + + for attempt in range(_MAX_SEQUENCE_RETRIES): + try: + seq = await _get_next_sequence_number(prisma_client, run_id, "events") + event_data: Dict[str, Any] = { + "run_id": run_id, + "event_type": data.event_type, + "step_name": data.step_name, + "sequence_number": seq, + } + if data.data is not None: + event_data["data"] = _json(data.data) + + async with prisma_client.db.tx() as tx: + event = await tx.litellm_workflowevent.create(data=event_data) + if new_status: + await tx.litellm_workflowrun.update( + where={"run_id": run_id}, + data={"status": new_status}, + ) + + return event + + except Exception as e: + if UniqueViolationError is not None and isinstance(e, UniqueViolationError): + if attempt == _MAX_SEQUENCE_RETRIES - 1: + verbose_proxy_logger.exception( + "Sequence number collision after %d retries for run %s", + _MAX_SEQUENCE_RETRIES, + run_id, + ) + raise HTTPException( + status_code=409, + detail="Concurrent write conflict — please retry", + ) + continue + verbose_proxy_logger.exception("Error appending workflow event: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + raise HTTPException( + status_code=500, detail="Failed to append event" + ) # pragma: no cover + + +@router.get( + "/v1/workflows/runs/{run_id}/events", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_workflow_events( + run_id: str, + limit: int = Query(100, ge=1, le=500), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Fetch event log for a run, ordered by sequence_number. Default limit 100, max 500.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + await _require_run(prisma_client, run_id, user_api_key_dict) + + try: + events = await prisma_client.db.litellm_workflowevent.find_many( + where={"run_id": run_id}, + order={"sequence_number": "asc"}, + take=limit, + ) + return {"events": events, "count": len(events)} + except Exception as e: + verbose_proxy_logger.exception("Error listing workflow events: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/v1/workflows/runs/{run_id}/messages", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def append_workflow_message( + run_id: str, + data: WorkflowMessageCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Append a conversation message. Stores full content (not truncated). + + Uses optimistic concurrency for sequence numbers. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + await _require_run(prisma_client, run_id, user_api_key_dict) + + for attempt in range(_MAX_SEQUENCE_RETRIES): + try: + seq = await _get_next_sequence_number(prisma_client, run_id, "messages") + msg_data: Dict[str, Any] = { + "run_id": run_id, + "role": data.role, + "content": data.content, + "sequence_number": seq, + } + if data.session_id is not None: + msg_data["session_id"] = data.session_id + msg = await prisma_client.db.litellm_workflowmessage.create(data=msg_data) + return msg + + except Exception as e: + if UniqueViolationError is not None and isinstance(e, UniqueViolationError): + if attempt == _MAX_SEQUENCE_RETRIES - 1: + verbose_proxy_logger.exception( + "Sequence number collision after %d retries for run %s", + _MAX_SEQUENCE_RETRIES, + run_id, + ) + raise HTTPException( + status_code=409, + detail="Concurrent write conflict — please retry", + ) + continue + verbose_proxy_logger.exception("Error appending workflow message: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + raise HTTPException( + status_code=500, detail="Failed to append message" + ) # pragma: no cover + + +@router.get( + "/v1/workflows/runs/{run_id}/messages", + tags=["workflow management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_workflow_messages( + run_id: str, + limit: int = Query(100, ge=1, le=500), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Fetch conversation history for a run, ordered by sequence_number. Default limit 100, max 500.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, detail=CommonProxyErrors.db_not_connected_error.value + ) + + await _require_run(prisma_client, run_id, user_api_key_dict) + + try: + messages = await prisma_client.db.litellm_workflowmessage.find_many( + where={"run_id": run_id}, + order={"sequence_number": "asc"}, + take=limit, + ) + return {"messages": messages, "count": len(messages)} + except Exception as e: + verbose_proxy_logger.exception("Error listing workflow messages: %s", e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 7599e11bde..d3b225e6e4 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -21,6 +21,28 @@ from litellm.proxy._types import ( from litellm.types.utils import StandardAuditLogPayload _audit_log_callback_cache: Dict[str, CustomLogger] = {} +ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY = "allow_litellm_changed_by_header" + + +def _allows_litellm_changed_by_header(user_api_key_dict: UserAPIKeyAuth) -> bool: + for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata): + if ( + isinstance(admin_metadata, dict) + and admin_metadata.get(ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY) is True + ): + return True + return False + + +def get_audit_log_changed_by( + *, + litellm_changed_by: Optional[str], + user_api_key_dict: UserAPIKeyAuth, + litellm_proxy_admin_name: Optional[str], +) -> Optional[str]: + if litellm_changed_by and _allows_litellm_changed_by_header(user_api_key_dict): + return litellm_changed_by + return user_api_key_dict.user_id or litellm_proxy_admin_name def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]: @@ -143,8 +165,10 @@ async def create_object_audit_log( if _store_audit_logs is not True: return - _changed_by = ( - litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name + _changed_by = get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, ) await create_audit_log_for_update( diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 410f636693..eb90d1b5ca 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -335,8 +335,9 @@ async def validate_key_mcp_servers_against_team( disallowed_servers = requested_servers - all_allowed_servers if disallowed_servers: if team_obj is not None: + team_id = team_obj.team_id detail = ( - f"Key requests MCP servers not allowed by team '{team_obj.team_id}': " + f"Key requests MCP servers not allowed by team '{team_id}': " f"{sorted(disallowed_servers)}. " f"Team allows: {sorted(team_allowed_servers)}. " f"Global (allow_all_keys) servers: {sorted(allow_all_keys_servers)}." @@ -365,8 +366,9 @@ async def validate_key_mcp_servers_against_team( disallowed_groups = requested_access_groups - team_access_groups if disallowed_groups: if team_obj is not None: + team_id = team_obj.team_id detail = ( - f"Key requests MCP access groups not allowed by team '{team_obj.team_id}': " + f"Key requests MCP access groups not allowed by team '{team_id}': " f"{sorted(disallowed_groups)}. " f"Team allows: {sorted(team_access_groups)}." ) @@ -390,13 +392,60 @@ async def validate_key_mcp_servers_against_team( if team_mcp_toolsets: disallowed_toolsets = requested_toolsets - set(team_mcp_toolsets) if disallowed_toolsets: + team_id = team_obj.team_id raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail={ "error": ( - f"Key requests MCP toolsets not allowed by team '{team_obj.team_id}': " + f"Key requests MCP toolsets not allowed by team '{team_id}': " f"{sorted(disallowed_toolsets)}. " f"Team allows: {sorted(team_mcp_toolsets)}." ) }, ) + + +def _extract_requested_search_tools(object_permission: Optional[dict]) -> List[str]: + """Return search_tool_name values from a key's object_permission dict.""" + if not object_permission or not isinstance(object_permission, dict): + return [] + raw = object_permission.get("search_tools") + if not isinstance(raw, list): + return [] + return [str(x) for x in raw if x] + + +async def validate_key_search_tools_against_team( + object_permission: Optional[dict], + team_obj: Optional["LiteLLM_TeamTableCachedObj"], +) -> None: + """ + Validate key object_permission.search_tools is a subset of the team's allowlist. + + Empty team allowlist means no restriction at team layer (skip). + """ + requested = _extract_requested_search_tools(object_permission) + if not requested: + return + + team_tools: List[str] = [] + if team_obj is not None and team_obj.object_permission is not None: + st = team_obj.object_permission.search_tools + if st: + team_tools = list(st) + + if not team_tools: + return + + disallowed = set(requested) - set(team_tools) + if disallowed: + team_id = team_obj.team_id if team_obj is not None else "unknown" + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + f"Key requests search tools not allowed by team '{team_id}': " + f"{sorted(disallowed)}. Team allows: {sorted(team_tools)}." + ) + }, + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 216eb61a9d..c42faa59cf 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -549,10 +549,16 @@ class AnthropicPassthroughLoggingHandler: # Create a mock user API key dict for the managed object storage from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + _request_metadata = (kwargs.get("litellm_params", {}) or {}).get( + "metadata", {} + ) or {} + user_api_key_dict = UserAPIKeyAuth( - user_id=kwargs.get("user_id", "default-user"), + user_id=_request_metadata.get( + "user_api_key_user_id", "default-user" + ), api_key="", - team_id=None, + team_id=_request_metadata.get("user_api_key_team_id"), team_alias=None, user_role=LitellmUserRoles.CUSTOMER, # Use proper enum value user_email=None, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 86dd23e12c..6a13853261 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -849,10 +849,16 @@ class VertexPassthroughLoggingHandler: # Create a mock user API key dict for the managed object storage from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + _request_metadata = (kwargs.get("litellm_params", {}) or {}).get( + "metadata", {} + ) or {} + user_api_key_dict = UserAPIKeyAuth( - user_id=kwargs.get("user_id", "default-user"), + user_id=_request_metadata.get( + "user_api_key_user_id", "default-user" + ), api_key="", - team_id=None, + team_id=_request_metadata.get("user_api_key_team_id"), team_alias=None, user_role=LitellmUserRoles.CUSTOMER, # Use proper enum value user_email=None, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 77eb3a5ee0..714b5f3c7b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -41,7 +41,6 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( - CommonProxyErrors, ConfigFieldInfo, ConfigFieldUpdate, LiteLLMRoutes, @@ -2325,12 +2324,14 @@ async def _register_pass_through_endpoint( dependencies = None if auth is not None and str(auth).lower() == "true": - if premium_user is not True: - raise ValueError( - "Error Setting Authentication on Pass Through Endpoint: {}".format( - CommonProxyErrors.not_premium_user.value - ) - ) + # Authentication on a pass-through endpoint used to be enterprise- + # only — which left the OSS tier with no safe configuration: the + # default was ``auth=False`` (unauthenticated forwarder) and the + # safe ``auth=True`` raised at startup unless the operator had a + # license. The default is now ``True`` (safe-by-default), and + # turning it on no longer requires a license: an unauthenticated + # forwarder is a deployment choice the operator should be allowed + # to make explicitly, but the safe option must always be free. dependencies = [Depends(user_api_key_auth)] if path not in LiteLLMRoutes.openai_routes.value: LiteLLMRoutes.openai_routes.value.append(path) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 302d7e76ed..cbfcd34c43 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -36,21 +36,16 @@ class PassThroughStreamingHandler: passthrough_success_handler_obj: PassThroughEndpointLogging, url_route: str, ): - """ - - Yields chunks from the response - - Collect non-empty chunks for post-processing (logging) - - Inject cost into chunks if include_cost_in_streaming_usage is enabled - """ - try: - raw_bytes: List[bytes] = [] - # Extract model name for cost injection - model_name = PassThroughStreamingHandler._extract_model_for_cost_injection( - request_body=request_body, - url_route=url_route, - endpoint_type=endpoint_type, - litellm_logging_obj=litellm_logging_obj, - ) + raw_bytes: List[bytes] = [] + logging_scheduled = False + model_name = PassThroughStreamingHandler._extract_model_for_cost_injection( + request_body=request_body, + url_route=url_route, + endpoint_type=endpoint_type, + litellm_logging_obj=litellm_logging_obj, + ) + try: async for chunk in response.aiter_bytes(): raw_bytes.append(chunk) if ( @@ -58,7 +53,6 @@ class PassThroughStreamingHandler: and model_name ): if endpoint_type == EndpointType.VERTEX_AI: - # Only handle streamRawPredict (uses Anthropic format) if "streamRawPredict" in url_route or "rawPredict" in url_route: modified_chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( chunk, model_name @@ -73,25 +67,32 @@ class PassThroughStreamingHandler: chunk = modified_chunk yield chunk - - # After all chunks are processed, handle post-processing - end_time = datetime.now() - - asyncio.create_task( - PassThroughStreamingHandler._route_streaming_logging_to_handler( - litellm_logging_obj=litellm_logging_obj, - passthrough_success_handler_obj=passthrough_success_handler_obj, - url_route=url_route, - request_body=request_body or {}, - endpoint_type=endpoint_type, - start_time=start_time, - raw_bytes=raw_bytes, - end_time=end_time, - ) - ) except Exception as e: verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}") raise + finally: + # GeneratorExit (raised on client disconnect) is not caught by + # `except Exception`; the finally block ensures partial usage + # still gets logged for spend tracking. See LIT-2642. + if not logging_scheduled and raw_bytes: + logging_scheduled = True + try: + asyncio.create_task( + PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body or {}, + endpoint_type=endpoint_type, + start_time=start_time, + raw_bytes=raw_bytes, + end_time=datetime.now(), + ) + ) + except Exception as e: + verbose_proxy_logger.error( + f"Error scheduling chunk_processor logging: {str(e)}" + ) @staticmethod async def _route_streaming_logging_to_handler( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8f676df04c..6cba6a3e96 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -91,6 +91,7 @@ from litellm.proxy._types import ( TeamDefaultSettings, TokenCountRequest, TransformRequestBody, + UI_TEAM_ID, UserAPIKeyAuth, ) from litellm.proxy.common_utils.callback_utils import ( @@ -235,37 +236,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( - router as mcp_byok_oauth_router, -) -from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - router as mcp_discoverable_endpoints_router, -) -from litellm.proxy._experimental.mcp_server.rest_endpoints import ( - router as mcp_rest_endpoints_router, -) -from litellm.proxy._experimental.mcp_server.server import app as mcp_app -from litellm.proxy._experimental.mcp_server.tool_registry import ( - global_mcp_tool_registry, -) from litellm.proxy._types import * -from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router -from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry -from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router -from litellm.proxy.agent_endpoints.model_list_helpers import ( - append_agents_to_model_group, - append_agents_to_model_info, -) +from litellm.proxy._lazy_features import attach_lazy_features from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) -from litellm.proxy.anthropic_endpoints.claude_code_endpoints import ( - claude_code_marketplace_router, -) -from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router -from litellm.proxy.anthropic_endpoints.skills_endpoints import ( - router as anthropic_skills_router, -) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, get_team_object, @@ -328,7 +303,6 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config from litellm.proxy.google_endpoints.endpoints import router as google_router -from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router from litellm.proxy.guardrails.init_guardrails import ( init_guardrails_v2, initialize_guardrails, @@ -344,9 +318,6 @@ from litellm.proxy.hooks.prompt_injection_detection import ( from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger from litellm.proxy.image_endpoints.endpoints import router as image_router from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request -from litellm.proxy.management_endpoints.access_group_endpoints import ( - router as access_group_router, -) from litellm.proxy.management_endpoints.budget_management_endpoints import ( router as budget_management_router, ) @@ -360,12 +331,6 @@ from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, admin_can_invite_user, ) -from litellm.proxy.management_endpoints.compliance_endpoints import ( - router as compliance_router, -) -from litellm.proxy.management_endpoints.config_override_endpoints import ( - router as config_override_router, -) from litellm.proxy.management_endpoints.cost_tracking_settings import ( router as cost_tracking_settings_router, ) @@ -379,9 +344,6 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) from litellm.proxy.management_endpoints.internal_user_endpoints import user_update -from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( - router as jwt_key_mapping_router, -) from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, @@ -390,9 +352,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, ) -from litellm.proxy.management_endpoints.mcp_management_endpoints import ( - router as mcp_management_router, -) from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( router as model_access_group_management_router, ) @@ -407,11 +366,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) -from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router from litellm.proxy.management_endpoints.router_settings_endpoints import ( router as router_settings_router, ) -from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router from litellm.proxy.management_endpoints.tag_management_endpoints import ( router as tag_management_router, ) @@ -423,15 +380,14 @@ from litellm.proxy.management_endpoints.team_endpoints import ( update_team, validate_membership, ) -from litellm.proxy.management_endpoints.tool_management_endpoints import ( - router as tool_management_router, +from litellm.proxy.management_endpoints.workflow_management_endpoints import ( + router as workflow_management_router, ) from litellm.proxy.memory.memory_endpoints import router as memory_router from litellm.proxy.management_endpoints.ui_sso import ( get_disabled_non_admin_personal_key_creation, ) from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router -from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( router as user_agent_analytics_router, ) @@ -441,7 +397,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import ( ) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router -from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) @@ -461,27 +416,16 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) -from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router -from litellm.proxy.policy_engine.policy_resolve_endpoints import ( - router as policy_resolve_router, -) -from litellm.proxy.prompts.prompt_endpoints import router as prompts_router from litellm.proxy.public_endpoints import router as public_endpoints_router from litellm.proxy.rag_endpoints.endpoints import router as rag_router -from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router from litellm.proxy.response_api_endpoints.endpoints import router as response_router from litellm.proxy.route_llm_request import route_request from litellm.proxy.search_endpoints.endpoints import router as search_router -from litellm.proxy.search_endpoints.search_tool_management import ( - router as search_tool_management_router, -) -from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router from litellm.proxy.spend_tracking.spend_management_endpoints import ( router as spend_management_router, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router from litellm.proxy.types_utils.utils import get_instance_fn from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( router as ui_crud_endpoints_router, @@ -511,16 +455,6 @@ from litellm.proxy.utils import ( prefetch_config_params, update_spend, ) -from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router -from litellm.proxy.vector_store_endpoints.management_endpoints import ( - router as vector_store_management_router, -) -from litellm.proxy.vector_store_files_endpoints.endpoints import ( - router as vector_store_files_router, -) -from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import ( - router as langfuse_router, -) from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.router import ( AssistantsTypedDict, @@ -742,6 +676,10 @@ async def _initialize_shared_aiohttp_session(): try: from aiohttp import ClientSession, TCPConnector + from litellm.llms.custom_httpx.http_handler import ( + _build_aiohttp_keepalive_socket_factory, + ) + connector_kwargs: Dict[str, Any] = { "keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT, "ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE, @@ -752,6 +690,9 @@ async def _initialize_shared_aiohttp_session(): connector_kwargs["limit"] = AIOHTTP_CONNECTOR_LIMIT if AIOHTTP_CONNECTOR_LIMIT_PER_HOST > 0: connector_kwargs["limit_per_host"] = AIOHTTP_CONNECTOR_LIMIT_PER_HOST + socket_factory = _build_aiohttp_keepalive_socket_factory() + if socket_factory is not None: + connector_kwargs["socket_factory"] = socket_factory connector = TCPConnector(**connector_kwargs) session = ClientSession(connector=connector) @@ -1093,6 +1034,11 @@ def get_openapi_schema(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + # Stub unloaded lazy features so they appear as Swagger sections. + from litellm.proxy._lazy_features import inject_lazy_stubs + + openapi_schema = inject_lazy_stubs(openapi_schema) + # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] @@ -1119,6 +1065,11 @@ def custom_openapi(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + # Stub unloaded lazy features so they appear as Swagger sections. + from litellm.proxy._lazy_features import inject_lazy_stubs + + openapi_schema = inject_lazy_stubs(openapi_schema) + # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] @@ -1546,14 +1497,78 @@ def mount_swagger_ui(): app.mount("/swagger", StaticFiles(directory=swagger_directory), name="swagger") + # On dropdown expand: one-time fetch to the prefix (triggers lazy load), + # then spec re-download so real routes replace the stub. Raw JS (no + #