tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-24 00:28:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into litellm_feat-add-guardrail

Browse Source
This commit is contained in:
Ryan Crabbe
2026-04-04 16:17:38 -07:00
parent 79064a68e4 51876292a0
commit c663926fb8
688 changed files with 17447 additions and 5537 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+69 -95
View File
@@ -39,7 +39,7 @@ commands:
# conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5)
pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \
"pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \
"pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \
"pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \
"responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \
"pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \
"a2a" "parameterized>=0.9.0"
@@ -150,16 +150,16 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \
"boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \
"azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
"boto3==1.42.80" langchain lunary==0.2.5 \
"azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
"detect_secrets==1.5.0" "respx==0.22.0" fastapi \
"gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \
"diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0"
"gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
"diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@@ -245,16 +245,16 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \
"boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \
"azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
"boto3==1.42.80" langchain lunary==0.2.5 \
"azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
"detect_secrets==1.5.0" "respx==0.22.0" fastapi \
"gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \
"diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0"
"gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
"diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@@ -346,42 +346,41 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
pip install "azure-identity==1.25.3"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install opentelemetry-api==1.28.0
pip install opentelemetry-sdk==1.28.0
pip install opentelemetry-exporter-otlp==1.28.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "httpx==0.28.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "apscheduler==3.10.4"
pip install "apscheduler==3.11.2"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "websockets==13.1.0"
pip install "Pillow==12.1.1"
pip install "jsonschema==4.23.0"
pip install "websockets==15.0.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
@@ -700,8 +699,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "pydantic==2.12.5"
pip install "mcp==1.26.0"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
@@ -746,7 +745,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "pydantic==2.12.5"
pip install "a2a-sdk"
# Run pytest and generate JUnit XML report
- run:
@@ -791,8 +790,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.36.0"
pip install "pydantic==2.12.5"
pip install "boto3==1.42.80"
pip install "semantic_router==0.1.10" --no-deps
pip install aurelio_sdk
pip install "pytest-xdist==3.6.1"
@@ -841,7 +840,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1048,8 +1047,8 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "pydantic==2.12.5"
pip install "mcp==1.26.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
@@ -1090,7 +1089,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
@@ -1137,7 +1136,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pytest-mock
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
@@ -1259,9 +1258,9 @@ jobs:
pip install pytest-mock
pip install "respx==0.22.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install "mlflow==2.17.2"
pip install "anthropic==0.52.0"
pip install "anthropic==0.54.0"
pip install "blockbuster==1.5.24"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
@@ -1352,11 +1351,11 @@ jobs:
pip install aiohttp
pip install openai
pip install click
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "tokenizers==0.22.2"
pip install "uvloop==0.21.0"
pip install "fastuuid==0.12.0"
pip install "fastuuid==0.14.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@@ -1389,7 +1388,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
pip install "mcp==1.25.0"
pip install "mcp==1.26.0"
- run:
name: Run tests
command: |
@@ -1679,10 +1678,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -1690,9 +1688,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -1823,10 +1821,9 @@ jobs:
pip install "mypy==1.18.2"
pip install "jsonlines==4.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@@ -1835,9 +1832,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -1969,10 +1966,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -1980,9 +1976,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -2611,16 +2607,16 @@ jobs:
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install aiohttp
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
@@ -2628,11 +2624,11 @@ jobs:
pip install fastapi
pip install jsonschema
pip install "httpx==0.27.0"
pip install "anyio==3.7.1"
pip install "anyio==4.8.0"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "google-cloud-aiplatform==1.59.0"
pip install "anthropic==0.52.0"
pip install "anthropic==0.54.0"
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
@@ -2799,7 +2795,7 @@ jobs:
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
@@ -3237,31 +3233,9 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
@@ -3535,7 +3509,7 @@ workflows:
only:
- main
- /litellm_.*/
# - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli
# - e2e_ui_testing:
# name: e2e_ui_testing_chromium
# browser: chromium
# context: e2e_ui_tests
.circleci/requirements.txt
+5 -5
View File
@@ -7,15 +7,15 @@ cohere
redis==5.2.1
redisvl==0.4.1
anthropic
orjson==3.10.12 # fast /embedding responses
pydantic==2.11.0
google-cloud-aiplatform==1.43.0
orjson==3.10.15 # fast /embedding responses
pydantic==2.12.5
google-cloud-aiplatform==1.133.0
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.25.0 # for MCP server
mcp==1.26.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
fastuuid==0.14.0
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries
litellm-proxy-extras # for prisma migrations
.devcontainer/post-create.sh
+1 -1
View File
@@ -12,6 +12,6 @@ echo "[post-create] Generating Prisma client"
poetry run prisma generate
echo "[post-create] Installing npm dependencies"
cd ui/litellm-dashboard && npm install --no-audit --no-fund
cd ui/litellm-dashboard && npm ci
echo "[post-create] Done"
.github/workflows/create-release.yml
+93
View File
@@ -0,0 +1,93 @@
name: Create Release
on:
workflow_dispatch:
inputs:
tag:
description: "Release tag (e.g. v1.83.0-stable)"
required: true
type: string
commit_hash:
description: "Full 40-char commit SHA to target"
required: true
type: string
permissions: {}
jobs:
release:
name: Create Release
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Validate inputs
env:
TAG: ${{ inputs.tag }}
COMMIT_HASH: ${{ inputs.commit_hash }}
run: |
if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
echo "::error::commit_hash must be a full 40-character commit SHA"
exit 1
fi
if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
echo "::error::tag must start with vX.Y.Z"
exit 1
fi
- name: Create release
env:
TAG: ${{ inputs.tag }}
COMMIT_HASH: ${{ inputs.commit_hash }}
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const tag = process.env.TAG;
const commitHash = process.env.COMMIT_HASH;
const cosignSection = [
`## Verify Docker Image Signature`,
``,
`All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
``,
'```bash',
`cosign verify \\`,
` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`,
` ghcr.io/berriai/litellm:${tag}`,
'```',
``,
`Expected output:`,
``,
'```',
`The following checks were performed on each of these signatures:`,
` - The cosign claims were validated`,
` - The signatures were verified against the specified public key`,
'```',
``,
`---`,
``,
].join('\n');
try {
const response = await github.rest.repos.createRelease({
draft: true,
generate_release_notes: true,
target_commitish: commitHash,
name: tag,
owner: context.repo.owner,
prerelease: false,
repo: context.repo.repo,
tag_name: tag,
});
const updatedBody = cosignSection + (response.data.body ?? '');
await github.rest.repos.updateRelease({
owner: context.repo.owner,
repo: context.repo.repo,
release_id: response.data.id,
body: updatedBody,
draft: false,
});
} catch (error) {
core.setFailed(error.message);
}
.github/workflows/scorecard.yml
+1 -1
View File
@@ -42,6 +42,6 @@ jobs:
retention-days: 5
- name: Upload to code scanning
uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
sarif_file: results.sarif
.github/workflows/test-litellm-matrix.yml
+41 -1
View File
@@ -171,4 +171,44 @@ jobs:
--reruns ${{ matrix.test-group.reruns }} \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \
--cov-config=pyproject.toml
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ matrix.test-group.name }}
path: coverage-${{ matrix.test-group.name }}.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: test
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # Required for OIDC tokenless upload
pull-requests: write # Required for Codecov PR comments
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Download all coverage reports
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-*
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false
.github/workflows/test-litellm-ui-build.yml
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
run: npm install
run: npm ci
- name: Build
run: npm run build
.gitignore
+1 -2
View File
@@ -72,8 +72,7 @@ tests/local_testing/log.txt
.codegpt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
.mypy_cache/*
.mypy_cache/*
**/.mypy_cache/
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
tests/llm_translation/test_vertex_key.json
.npmrc
+5
View File
@@ -0,0 +1,5 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
.pre-commit-config.yaml
-40
View File
@@ -1,40 +0,0 @@
repos:
- repo: local
hooks:
- id: pyright
name: pyright
entry: pyright
language: system
types: [python]
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- id: black
name: black
entry: poetry run black
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system
Dockerfile
+5 -5
View File
@@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@@ -15,7 +15,7 @@ USER root
# Install build dependencies
RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
RUN python -m pip install build
RUN python -m pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@@ -90,7 +90,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
ci_cd/publish-proxy-extras.sh
-19
View File
@@ -1,19 +0,0 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"
ci_cd/security_scans.sh
-1
View File
@@ -160,7 +160,6 @@ run_grype_scans() {
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
codecov.yaml
+3
View File
@@ -17,6 +17,9 @@ component_management:
- component_id: "Proxy_Authentication"
paths:
- "*/proxy/auth/**"
- component_id: "Enterprise"
paths:
- "enterprise/**"
comment:
layout: "header, diff, flags, components" # show component info in the PR comment
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
+1 -1
View File
@@ -230,7 +230,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
cookbook/misc/test_responses_api.py
+2 -2
View File
@@ -20,7 +20,7 @@ base64_image = encode_image(image_path)
response = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
input=[
{
"role": "user",
@@ -43,7 +43,7 @@ print("sleeping for 20 seconds...")
time.sleep(20)
print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
)
cosign.pub
+4
View File
@@ -0,0 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
-----END PUBLIC KEY-----
deploy/Dockerfile.ghcr_base
+1 -1
View File
@@ -1,5 +1,5 @@
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest
FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
# Set the working directory to /app
WORKDIR /app
docker/Dockerfile.alpine
+5 -5
View File
@@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@@ -13,8 +13,8 @@ WORKDIR /app
# Install build dependencies
RUN apk add --no-cache gcc python3-dev musl-dev
RUN pip install --upgrade pip && \
pip install build
RUN pip install --upgrade pip==26.0.1 && \
pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@@ -44,7 +44,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
docker/Dockerfile.custom_ui
+4 -3
View File
@@ -1,4 +1,5 @@
# Use the provided base image
# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
# Set the working directory to /app
@@ -18,8 +19,8 @@ RUN apt-get update && apt-get upgrade -y \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
apt-get install -y --no-install-recommends nodejs npm && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -51,7 +52,7 @@ ENV UI_BASE_PATH="/prod/ui"
# Build the UI with the specified UI_BASE_PATH
WORKDIR /app/ui/litellm-dashboard
RUN npm install
RUN npm ci
RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
# Create the destination directory
docker/Dockerfile.database
+5 -5
View File
@@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@@ -21,7 +21,7 @@ RUN apk add --no-cache \
openssl \
openssl-dev
RUN python -m pip install build
RUN python -m pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -82,7 +82,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
docker/Dockerfile.dev
+5 -5
View File
@@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-slim
ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@@ -19,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libssl-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/* \
&& pip install --upgrade pip build
&& pip install --upgrade pip==26.0.1 build==1.4.2
# Copy requirements first for better layer caching
COPY requirements.txt .
@@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -109,7 +109,7 @@ COPY --from=builder /wheels/ /wheels/
COPY --from=builder /app/dist/*.whl .
# Install all dependencies in one step with no-cache for smaller image
RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && \
rm -f *.whl && \
rm -rf /wheels
docker/Dockerfile.health_check
+9 -1
View File
@@ -1,4 +1,4 @@
FROM python:3.11-slim
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
@@ -12,5 +12,13 @@ RUN pip install --no-cache-dir -r requirements.txt
# Make script executable
RUN chmod +x /app/health_check_client.py
# Run as non-root user
RUN adduser --disabled-password --gecos "" --uid 1001 healthcheck
USER healthcheck
# Health check
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
CMD python /app/health_check_client.py --help || exit 1
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]
docker/Dockerfile.non_root
+18 -6
View File
@@ -1,6 +1,6 @@
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
ARG PROXY_EXTRAS_SOURCE=published
# -----------------
@@ -27,7 +27,7 @@ RUN for i in 1 2 3; do \
nodejs \
npm && break || sleep 5; \
done \
&& pip install --no-cache-dir --upgrade pip build
&& pip install --no-cache-dir --upgrade pip==26.0.1 build==1.4.2
# Cache Python dependencies
COPY requirements.txt .
@@ -41,13 +41,22 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI using the upstream command order while keeping a single RUN layer
# NOTE: .npmrc (which has ignore-scripts=true and min-release-age=3d) is temporarily
# renamed during npm install/ci. This is safe because npm ci installs from
# package-lock.json with pinned versions + integrity hashes.
RUN mkdir -p /var/lib/litellm/ui && \
npm install -g npm@latest && npm cache clean --force && \
mv /app/.npmrc /app/.npmrc.bak && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
npm install --legacy-peer-deps && \
mv .npmrc .npmrc.bak && \
npm ci && \
mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
mkdir -p /var/lib/litellm/assets && \
@@ -106,7 +115,7 @@ RUN for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -133,6 +142,9 @@ COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/
# Keep enterprise bridge module in runtime so `enterprise.enterprise_hooks`
# can load and register managed enterprise hooks (e.g. managed_files).
COPY --from=builder /app/enterprise /app/enterprise
# Copy prisma_migration.py for Helm migrations job compatibility
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
COPY --from=builder /wheels/ /wheels/
docker/build_admin_ui.sh
+13 -2
View File
@@ -40,11 +40,22 @@ else
exit 1
fi
fi
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
NVM_VERSION="v0.40.4"
NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f"
NVM_SCRIPT=$(mktemp)
trap 'rm -f "$NVM_SCRIPT"' EXIT
curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT"
if command -v sha256sum &>/dev/null; then
echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c -
elif command -v shasum &>/dev/null; then
echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c -
else
echo "No sha256 tool found; cannot verify nvm checksum"; exit 1
fi || { echo "nvm checksum verification failed"; exit 1; }
bash "$NVM_SCRIPT"
source ~/.nvm/nvm.sh
nvm install v18.17.0
nvm use v18.17.0
npm install -g npm
# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
docker/build_from_pip/Dockerfile.build_from_pip
+7 -9
View File
@@ -1,19 +1,17 @@
FROM python:3.13-alpine
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
ENV HOME=/home/litellm
ENV PATH="${HOME}/venv/bin:$PATH"
# Install runtime dependencies
# Note: Using Python 3.13 for compatibility with ddtrace and other packages
# rust and cargo are required for building ddtrace from source
# musl-dev and libffi-dev are needed for some Python packages on Alpine
RUN apk update && \
apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo
# Install runtime dependencies needed for building native extensions
RUN apt-get update && \
apt-get install -y --no-install-recommends gcc libffi-dev && \
rm -rf /var/lib/apt/lists/*
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip==26.0.1
COPY docker/build_from_pip/requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
@@ -28,4 +26,4 @@ RUN prisma generate
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]
docker/build_from_pip/requirements.txt
+5 -5
View File
@@ -1,6 +1,6 @@
litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
litellm[proxy]==1.83.0
prometheus_client==0.20.0
langfuse==2.59.7
prisma==0.11.0
openai==2.24.0
ddtrace==2.19.0 # for advanced DD tracing / profiling
docker/install_auto_router.sh
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19
pip install aurelio-sdk==0.0.19 --no-deps
docs/my-website/blog/april_townhall_announcement/index.md
+39
View File
@@ -0,0 +1,39 @@
---
slug: april-townhall-announcement
title: "April Townhall: Security + Product Roadmap"
date: 2026-04-02T07:30:00
authors:
- krrish
description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
tags: [announcement, townhall]
hide_table_of_contents: true
---
import Image from '@theme/IdealImage';
We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
<Image
img={require('../../img/april_townhall_banner.png')}
style={{width: '900px', height: 'auto', display: 'block'}}
/>
{/* truncate */}
## Agenda
- Product updates and roadmap progress
- Reliability and security updates
- Open Q&A with the team
## How to contribute
Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
## Register
Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.
docs/my-website/blog/security_hardening_april_2026/index.md
+66
View File
@@ -0,0 +1,66 @@
---
slug: security-hardening-april-2026
title: "Security Update: Vulnerability Disclosures and Ongoing Hardening"
date: 2026-04-03T12:00:00
authors:
- krrish
- ishaan-alt
description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program."
tags: [security]
hide_table_of_contents: false
---
After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading.
We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions.
The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users.
The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.**
{/* truncate */}
## Vulnerabilities
### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical)
Found by Veria Labs.
When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead.
**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround.
Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)
### CVE-2026-35029: Privilege escalation via `/config/update` (High)
Found by Lakera.
`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint.
Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789)
### Password hash exposure and pass-the-hash login (High)
Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/).
Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses.
Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)
## Bug bounty program
After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues.
Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities:
| Severity | Bounty | Example |
|----------|--------|---------|
| Critical | $1,500 – $3,000 | Supply chain compromise |
| High | $500 – $1,500 | Unauthenticated access to protected data |
We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security).
## What's next
Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed.
docs/my-website/docs/anthropic_count_tokens.md
+1 -1
View File
@@ -96,7 +96,7 @@ model_list:
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: us-west-2
```
docs/my-website/docs/completion/computer_use.md
+3 -3
View File
@@ -80,7 +80,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
@@ -153,7 +153,7 @@ import litellm
assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True
assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
assert litellm.supports_computer_use(model="openai/gpt-4") == False
```
@@ -171,7 +171,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
docs/my-website/docs/completion/document_understanding.md
+9 -9
View File
@@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@@ -63,7 +63,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@@ -153,7 +153,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@@ -242,7 +242,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
<Tabs>
<TabItem label="SDK" value="sdk">
Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input
Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input
```python
assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True
assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True
```
</TabItem>
@@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202
model_list:
- model_name: bedrock-model # model group name
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
docs/my-website/docs/completion/provider_specific_params.md
+1 -1
View File
@@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
requestMetadata={"cost_center": "engineering"}
)
docs/my-website/docs/index.md
+1 -1
View File
@@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret"
os.environ["AWS_REGION_NAME"] = "us-east-1"
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001:0",
messages=[{"role": "user", "content": "Hello, how are you?"}]
)
print(response.choices[0].message.content)
docs/my-website/docs/mcp.md
+2 -1
View File
@@ -278,7 +278,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
aws_service_name: bedrock-agentcore
docs/my-website/docs/mcp_aws_sigv4.md
+51 -2
View File
@@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r
| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
| **AWS Session Token** | No | Only needed for temporary STS credentials |
| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing |
| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted |
Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
@@ -66,8 +68,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended)
aws_session_name: "litellm-prod" # optional — for CloudTrail auditing
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
@@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing |
| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` |
## How It Works
@@ -157,6 +161,42 @@ mcp_servers:
aws_service_name: "bedrock-agentcore"
```
## Using IAM Role Assumption (AssumeRole)
For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests:
```yaml title="config.yaml with AssumeRole" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole"
aws_session_name: "litellm-prod" # optional
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials.
You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call:
```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: os.environ/AWS_ROLE_ARN
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: "us-east-1"
```
:::tip
For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically.
:::
## Troubleshooting
### 403 Forbidden from AWS
@@ -166,6 +206,15 @@ mcp_servers:
- Ensure `aws_service_name` is set to `bedrock-agentcore`
- If using STS credentials, confirm `aws_session_token` is set and not expired
### AssumeRole AccessDenied
If you get `AccessDenied` when using `aws_role_name`:
- Verify the role ARN is correct
- Check that the trust policy on the target role allows your source identity to assume it
- If running on EKS, ensure the pod's service account is annotated with the correct IAM role
- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error
### Health check errors on startup
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
docs/my-website/docs/oidc.md
+1 -1
View File
@@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure
model list:
- model_name: aws/claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: "eu-central-1"
aws_role_name: "arn:aws:iam::12345678:role/bedrock-role"
aws_web_identity_token: "oidc/azure/api://123-456-789-9d04"
docs/my-website/docs/providers/azure/azure_responses.md
+2 -2
View File
@@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint.
from litellm import completion
import os
os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/"
os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
os.environ["AZURE_API_KEY"] = "my-api-key"
@@ -268,7 +268,7 @@ model_list:
litellm_params:
model: azure/responses/my-custom-o1-pro
api_key: os.environ/AZURE_API_KEY
api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
api_base: https://my-azure-endpoint.openai.azure.com/
api_version: 2023-03-15-preview
```
docs/my-website/docs/providers/bedrock.md
+10 -10
View File
@@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server
model_list:
- model_name: bedrock-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
requestMetadata={
"cost_center": "engineering",
@@ -354,7 +354,7 @@ response = completion(
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
requestMetadata:
cost_center: "engineering"
```
@@ -1543,7 +1543,7 @@ file_data = response.content
encoded_file = base64.b64encode(file_data).decode("utf-8")
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@@ -1574,7 +1574,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@@ -1660,7 +1660,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
@@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
)
@@ -2068,7 +2068,7 @@ print(response)
model_list:
- model_name: anthropic-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
# You have to set the ARN application inference profile in the model_id parameter
model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0
```
docs/my-website/docs/providers/bedrock_image_gen.md
+23 -1
View File
@@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Amazon Nova Canvas - Image Edit
Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html):
| Scenario | `taskType` sent to Bedrock |
|----------|----------------------------|
| Image + prompt (no mask) | `IMAGE_VARIATION` |
| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) |
| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) |
| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` |
```python
from litellm import image_edit
response = image_edit(
image=open("photo.png", "rb"),
prompt="Add soft sunset lighting",
model="bedrock/amazon.nova-canvas-v1:0",
)
```
For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`).
## Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
@@ -147,4 +170,3 @@ model_list:
## Authentication
All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.
docs/my-website/docs/providers/oci.md
+105 -4
View File
@@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ
## Supported Models
### Meta Llama Models
### Chat / Text Generation
#### Meta Llama Models
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
- `meta.llama-4-scout-17b-16e-instruct`
- `meta.llama-3.3-70b-instruct`
- `meta.llama-3.3-70b-instruct-fp8-dynamic`
- `meta.llama-3.2-90b-vision-instruct`
- `meta.llama-3.2-11b-vision-instruct`
- `meta.llama-3.1-405b-instruct`
- `meta.llama-3.1-70b-instruct`
### xAI Grok Models
#### xAI Grok Models
- `xai.grok-4.20`
- `xai.grok-4.20-multi-agent`
- `xai.grok-4`
- `xai.grok-4-fast`
- `xai.grok-4.1-fast`
- `xai.grok-3`
- `xai.grok-3-fast`
- `xai.grok-3-mini`
- `xai.grok-3-mini-fast`
- `xai.grok-code-fast-1`
### Cohere Models
#### Cohere Models
- `cohere.command-latest`
- `cohere.command-a-03-2025`
- `cohere.command-a-reasoning-08-2025`
- `cohere.command-a-vision-07-2025`
- `cohere.command-a-translate-08-2025`
- `cohere.command-plus-latest`
- `cohere.command-r-08-2024`
- `cohere.command-r-plus-08-2024`
#### Google Gemini Models (via OCI)
- `google.gemini-2.5-pro`
- `google.gemini-2.5-flash`
- `google.gemini-2.5-flash-lite`
### Embedding Models
- `cohere.embed-english-v3.0` (1024 dimensions)
- `cohere.embed-english-light-v3.0` (384 dimensions)
- `cohere.embed-multilingual-v3.0` (1024 dimensions)
- `cohere.embed-multilingual-light-v3.0` (384 dimensions)
- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal)
- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-v4.0` (1536 dimensions, multimodal)
## Authentication
@@ -394,4 +424,75 @@ response = completion(
| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy |
| `oci_key` | string | - | (Manual auth) The private key content as a string |
| `oci_key_file` | string | - | (Manual auth) Path to the private key file |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
## Embeddings
LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above.
<Tabs>
<TabItem value="embed-manual" label="Manual Credentials" default>
```python
from litellm import embedding
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_region="us-ashburn-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```
</TabItem>
<TabItem value="embed-sdk" label="OCI SDK Signer">
```python
from litellm import embedding
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_signer=signer,
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
</Tabs>
### Embedding Optional Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` |
| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` |
### Using Dedicated Embedding Endpoints
```python
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world"],
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...",
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
# ... auth params
)
```
docs/my-website/docs/proxy/config_settings.md
+4
View File
@@ -201,6 +201,7 @@ router_settings:
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
| default_team_params | object | Default parameters applied to every new team created via `/team/new` (including SSO auto-created teams). Only fills in fields not explicitly set in the request. Sub-fields: `max_budget` (float), `budget_duration` (string, e.g. `"30d"`), `tpm_limit` (integer), `rpm_limit` (integer), `team_member_permissions` (array of strings, e.g. `["/team/daily/activity", "/key/generate"]`), `models` (array of strings — only applied to SSO auto-created teams). |
### general_settings - Reference
@@ -288,6 +289,7 @@ router_settings:
| database_connection_pool_timeout | integer | Database connection pool timeout in seconds |
| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database |
| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry |
| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations |
| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls |
@@ -396,6 +398,7 @@ router_settings:
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments |
| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
### environment variables - Reference
@@ -820,6 +823,7 @@ router_settings:
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS | TTL in seconds for the distributed lock used by the key rotation job. Default is 600 (10 minutes).
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
docs/my-website/docs/proxy/guardrails/guardrail_policies.md
+2 -2
View File
@@ -311,7 +311,7 @@ Response:
## Policy Flow Builder
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step pass/fail actions.
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step **pass**, **fail**, and optional **error** actions (`on_pass`, `on_fail`, `on_error`).
## Config Reference
@@ -337,7 +337,7 @@ policies:
| `guardrails.add` | `list[string]` | Guardrails to enable. |
| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions. See [Policy Flow Builder](./policy_flow_builder). |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions (`on_pass`, `on_fail`, optional `on_error`). See [Policy Flow Builder](./policy_flow_builder). |
### `policy_attachments`
docs/my-website/docs/proxy/guardrails/policy_flow_builder.md
+55 -10
View File
@@ -1,8 +1,8 @@
# Policy Flow Builder
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail passes or fails.
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail **passes**, **fails a policy check** (content intervention), or hits a **technical error** (e.g. timeout, unreachable provider, missing guardrail).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors). With **`on_error`**, you can treat **technical** failures differently from **policy** failures—for example, fall back to another provider when the primary API errors, while still blocking on flagged content.
## When to use the Flow Builder
@@ -19,6 +19,7 @@ Use the Flow Builder when you need:
- **Custom responses** — return a specific message when a guardrail fails instead of a generic block
- **Data chaining** — pass modified data (e.g., PII-masked content) from one step to the next
- **Fine-grained control** — different actions on pass vs. fail per step
- **Technical-error routing** — set `on_error` separately from `on_fail` so outages or timeouts can **allow**, **block**, **go to the next step**, or return a **custom response** without conflating them with content violations
## Concepts
@@ -29,24 +30,37 @@ A pipeline has:
- **Mode**: `pre_call` (before the LLM) or `post_call` (after the LLM)
- **Steps**: Ordered list of guardrail steps
### Outcomes: pass, fail, and error
Each step run produces one of three outcomes:
| Outcome | Meaning | Typical cause |
|--------|---------|----------------|
| **pass** | Guardrail completed without blocking | Content allowed, or data was modified and returned |
| **fail** | Policy intervention | Guardrail raised an intervention (e.g. flagged content, blocked request) |
| **error** | Technical failure | Timeouts, network errors, guardrail not registered, or other non-intervention exceptions |
`on_pass` and `on_fail` apply to **pass** and **fail** respectively. **`on_error`** applies only to **error**. If `on_error` is omitted, the pipeline uses **`on_fail`** for error outcomes (backward compatible).
### Step actions
Each step defines what happens when the guardrail **passes** and when it **fails**:
For each step you choose an action for **pass**, **fail**, and optionally **error**. Allowed values are: `next`, `allow`, `block`, `modify_response`.
| Action | Description |
|--------|-------------|
| **Next Step** | Continue to the next guardrail in the pipeline |
| **Allow** | Stop the pipeline and allow the request to proceed |
| **Block** | Stop the pipeline and block the request |
| **Custom Response** | Return a custom message instead of the default block |
| **Next Step** (`next`) | Continue to the next guardrail in the pipeline |
| **Allow** (`allow`) | Stop the pipeline and allow the request to proceed |
| **Block** (`block`) | Stop the pipeline and block the request |
| **Custom Response** (`modify_response`) | Return a custom message instead of the default block |
### Step options
| Field | Type | Description |
|-------|------|--------------|
| `guardrail` | `string` | Name of the guardrail to run |
| `on_pass` | `string` | Action when guardrail passes: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when guardrail fails: `next`, `allow`, `block`, `modify_response` |
| `on_pass` | `string` | Action when outcome is **pass**: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when outcome is **fail** (policy intervention): `next`, `allow`, `block`, `modify_response` |
| `on_error` | `string` (optional) | Action when outcome is **error** (technical). If omitted, **error** uses `on_fail`. |
| `pass_data` | `boolean` | Forward modified request data (e.g., PII-masked) to the next step |
| `modify_response_message` | `string` | Custom message when using `modify_response` action |
@@ -57,7 +71,7 @@ Each step defines what happens when the guardrail **passes** and when it **fails
3. Select **Flow Builder** (instead of the simple form)
4. Design your flow:
- **Trigger** — Incoming LLM request (runs when the policy matches)
- **Steps** — Add guardrails, set ON PASS and ON FAIL actions per step
- **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL)
- **End** — Request proceeds to the LLM
5. Use the **+** between steps to insert new steps
6. Use the **Test** panel to run sample messages through the pipeline before saving
@@ -151,6 +165,37 @@ policies:
First attempt passes → allow. First attempt fails → retry the same guardrail; second pass → allow, second fail → block.
## Technical errors vs policy failures (`on_error`)
Use **`on_error`** when you want different behavior for **API/infra problems** than for **content policy** violations.
- **`on_fail`** — Runs when the guardrail **intervenes** (e.g. toxic content, PII detected).
- **`on_error`** — Runs when the step ends in **error** (timeout, connection failure, guardrail not loaded, etc.). If you omit `on_error`, **error** outcomes use **`on_fail`**.
Example: block on bad content, but if the primary scanner is down, fall back to a second guardrail instead of blocking every request:
```yaml
policies:
error-fallback-policy:
guardrails:
add:
- primary_scanner
- backup_scanner
pipeline:
mode: pre_call
steps:
- guardrail: primary_scanner
on_pass: allow
on_fail: block
on_error: next
- guardrail: backup_scanner
on_pass: allow
on_fail: block
on_error: allow
```
If `primary_scanner` errors → run `backup_scanner`. If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed).
## Example: Custom response on fail
Return a branded message instead of a generic block:
docs/my-website/docs/proxy/health.md
+2 -79
View File
@@ -316,86 +316,9 @@ general_settings:
## Health Check Driven Routing
By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown.
Route traffic away from unhealthy deployments proactively — before user requests hit them. Supports per-error-type failure thresholds, transient error suppression, and automatic safety nets.
With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown.
### How it works
1. Background health checks run on their configured interval
2. After each cycle, every deployment is marked healthy or unhealthy
3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing
4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage)
5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored
### Quick start
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true
```
### Configuration
| Setting | Where | Default | Description |
|---------|-------|---------|-------------|
| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing |
| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles |
### Interaction with cooldown
Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism:
- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check
- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL
This means request failures still provide fast detection between health check intervals.
### Staleness
If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down.
The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s.
### Example: custom staleness
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_staleness_threshold: 90 # ignore health state older than 90s
```
### Debugging
Run the proxy with `--detailed_debug` and look for:
```
health_check_routing_state_updated healthy=3 unhealthy=1
```
This is logged after each health check cycle when routing state is written.
If the safety net triggers (all deployments unhealthy), you'll see:
```
All deployments marked unhealthy by health checks, bypassing health filter
```
See the full guide: [Health Check Driven Routing](./health_check_routing.md)
## Health Check Timeout
docs/my-website/docs/proxy/health_check_routing.md
+340
View File
@@ -0,0 +1,340 @@
# Health Check Driven Routing
Route traffic away from unhealthy deployments before users hit errors. Background health checks run on a configurable interval, and any deployment that fails gets removed from the routing pool proactively, not after a user request already failed.
## Architecture
<svg viewBox="0 0 860 600" xmlns="http://www.w3.org/2000/svg" style={{maxWidth: '100%', fontFamily: 'system-ui, sans-serif'}}>
{/* Background */}
<rect width="860" height="600" fill="#f8fafc" rx="12"/>
{/* LEFT PANEL: Background health check loop */}
<rect x="20" y="20" width="240" height="560" fill="#eff6ff" rx="10" stroke="#bfdbfe" strokeWidth="1.5"/>
<text x="140" y="48" textAnchor="middle" fill="#1d4ed8" fontSize="13" fontWeight="600">Background Loop</text>
<text x="140" y="64" textAnchor="middle" fill="#3b82f6" fontSize="11">every health_check_interval seconds</text>
{/* Deployment A */}
<rect x="40" y="82" width="200" height="50" fill="white" rx="8" stroke="#93c5fd" strokeWidth="1.5"/>
<text x="140" y="102" textAnchor="middle" fill="#1e40af" fontSize="12" fontWeight="500">Deployment A</text>
<text x="140" y="120" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 200 ✓</text>
{/* Deployment B */}
<rect x="40" y="148" width="200" height="50" fill="white" rx="8" stroke="#fca5a5" strokeWidth="1.5"/>
<text x="140" y="168" textAnchor="middle" fill="#991b1b" fontSize="12" fontWeight="500">Deployment B</text>
<text x="140" y="186" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 401 ✗</text>
{/* Deployment C */}
<rect x="40" y="214" width="200" height="50" fill="white" rx="8" stroke="#fde68a" strokeWidth="1.5"/>
<text x="140" y="234" textAnchor="middle" fill="#92400e" fontSize="12" fontWeight="500">Deployment C</text>
<text x="140" y="252" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 429 ⚡</text>
{/* ignore_transient box */}
<rect x="40" y="282" width="200" height="68" fill="#fefce8" rx="8" stroke="#fde047" strokeWidth="1.5"/>
<text x="140" y="302" textAnchor="middle" fill="#713f12" fontSize="11" fontWeight="600">ignore_transient_errors: true</text>
<text x="140" y="320" textAnchor="middle" fill="#92400e" fontSize="11">429 / 408 → ignored</text>
<text x="140" y="338" textAnchor="middle" fill="#92400e" fontSize="11">not written to cache</text>
{/* allowed_fails_policy box */}
<rect x="40" y="368" width="200" height="84" fill="#f0fdf4" rx="8" stroke="#86efac" strokeWidth="1.5"/>
<text x="140" y="388" textAnchor="middle" fill="#166534" fontSize="11" fontWeight="600">allowed_fails_policy</text>
<text x="140" y="406" textAnchor="middle" fill="#15803d" fontSize="11">401 → increment counter</text>
<text x="140" y="424" textAnchor="middle" fill="#15803d" fontSize="11">counter &gt; threshold</text>
<text x="140" y="442" textAnchor="middle" fill="#15803d" fontSize="11">→ cooldown triggered</text>
{/* CENTER PANEL: Shared State */}
<rect x="300" y="20" width="220" height="560" fill="#f5f3ff" rx="10" stroke="#c4b5fd" strokeWidth="1.5"/>
<text x="410" y="48" textAnchor="middle" fill="#6d28d9" fontSize="13" fontWeight="600">Shared State</text>
{/* Health State Cache */}
<rect x="320" y="62" width="180" height="116" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="84" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">DeploymentHealthCache</text>
<text x="410" y="104" textAnchor="middle" fill="#64748b" fontSize="11">A → healthy ✓</text>
<text x="410" y="122" textAnchor="middle" fill="#64748b" fontSize="11">B → unhealthy ✗</text>
<text x="410" y="140" textAnchor="middle" fill="#64748b" fontSize="11">C → not written (ignored)</text>
<text x="410" y="164" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: staleness_threshold × 1.5</text>
{/* Cooldown Cache */}
<rect x="320" y="196" width="180" height="104" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="218" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">Cooldown Cache</text>
<text x="410" y="238" textAnchor="middle" fill="#64748b" fontSize="11">B → cooling down</text>
<text x="410" y="256" textAnchor="middle" fill="#64748b" fontSize="11">(after policy threshold)</text>
<text x="410" y="278" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time</text>
{/* failed_calls counter */}
<rect x="320" y="318" width="180" height="90" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="340" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">failed_calls counter</text>
<text x="410" y="360" textAnchor="middle" fill="#64748b" fontSize="11">B: 2 / AuthAllowedFails: 1</text>
<text x="410" y="378" textAnchor="middle" fill="#64748b" fontSize="11">→ threshold exceeded</text>
<text x="410" y="398" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time (must &gt; interval)</text>
{/* RIGHT PANEL: Request path */}
<rect x="560" y="20" width="280" height="560" fill="#fff7ed" rx="10" stroke="#fed7aa" strokeWidth="1.5"/>
<text x="700" y="48" textAnchor="middle" fill="#c2410c" fontSize="13" fontWeight="600">Request Path</text>
{/* Incoming request */}
<rect x="580" y="62" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="85" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="500">Incoming request</text>
{/* All deployments */}
<rect x="580" y="120" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="143" textAnchor="middle" fill="#9a3412" fontSize="12">All deployments [A, B, C]</text>
<line x1="700" y1="100" x2="700" y2="120" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Health check filter */}
<rect x="580" y="178" width="240" height="62" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="200" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">① Health Check Filter</text>
<text x="700" y="218" textAnchor="middle" fill="#64748b" fontSize="11">if policy set → bypass</text>
<text x="700" y="234" textAnchor="middle" fill="#64748b" fontSize="11">else → remove unhealthy</text>
<line x1="700" y1="158" x2="700" y2="178" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Cooldown filter */}
<rect x="580" y="262" width="240" height="50" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="284" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">② Cooldown Filter</text>
<text x="700" y="302" textAnchor="middle" fill="#64748b" fontSize="11">remove deployments in cooldown</text>
<line x1="700" y1="240" x2="700" y2="262" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Safety net */}
<rect x="580" y="334" width="240" height="52" fill="#fef9c3" rx="7" stroke="#fbbf24" strokeWidth="1.5"/>
<text x="700" y="356" textAnchor="middle" fill="#713f12" fontSize="12" fontWeight="600">Safety Net</text>
<text x="700" y="376" textAnchor="middle" fill="#713f12" fontSize="11">if all removed → return all</text>
<line x1="700" y1="312" x2="700" y2="334" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Load balancer */}
<rect x="580" y="408" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="431" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">③ Load Balancer</text>
<line x1="700" y1="386" x2="700" y2="408" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Selected deployment */}
<rect x="580" y="468" width="240" height="38" fill="#dcfce7" rx="7" stroke="#4ade80" strokeWidth="1.5"/>
<text x="700" y="491" textAnchor="middle" fill="#14532d" fontSize="12" fontWeight="600">Selected: Deployment A ✓</text>
<line x1="700" y1="446" x2="700" y2="468" stroke="#4ade80" strokeWidth="1.5" markerEnd="url(#arrow-green)"/>
{/* ARROWS: left → center */}
<line x1="240" y1="107" x2="320" y2="110" stroke="#3b82f6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-blue)"/>
<line x1="240" y1="173" x2="320" y2="240" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="173" x2="320" y2="348" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="316" x2="320" y2="130" stroke="#eab308" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-yellow)"/>
{/* ARROWS: center → right */}
<line x1="500" y1="120" x2="580" y2="190" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
<line x1="500" y1="248" x2="580" y2="274" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
{/* Arrow markers */}
<defs>
<marker id="arrow-orange" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#fb923c"/>
</marker>
<marker id="arrow-blue" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#3b82f6"/>
</marker>
<marker id="arrow-red" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#ef4444"/>
</marker>
<marker id="arrow-yellow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#eab308"/>
</marker>
<marker id="arrow-purple" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#8b5cf6"/>
</marker>
<marker id="arrow-green" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#4ade80"/>
</marker>
</defs>
</svg>
## What problem does this solve?
By default, LiteLLM routes traffic to all deployments and only stops sending to a broken one after it has already failed a user request. The cooldown system is reactive.
Health check driven routing makes this **proactive**: a background loop pings every deployment on a configurable interval. If a deployment fails its health check, it gets removed from the routing pool immediately, before a user request lands on it.
When you also set `allowed_fails_policy`, you control exactly how many health check failures of each error type (auth errors, rate limits, timeouts) are needed before a deployment enters cooldown. This avoids false positives from transient noise.
## Setup
### Step 1: Enable background health checks
Background health checks are off by default. Turn them on in `general_settings`:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60 # seconds between each full check cycle
```
### Step 2: Enable health check routing
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true # ← route away from unhealthy deployments
```
At this point, any deployment that fails its health check is immediately excluded from routing until the next check cycle clears it.
### Step 3: Add a policy to control how many failures trigger cooldown
Without a policy, the first health check failure marks a deployment as unhealthy. If you want more tolerance (e.g., only act after 2 consecutive auth failures), use `allowed_fails_policy`:
```yaml
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
router_settings:
cooldown_time: 60 # how long a deployment stays in cooldown
allowed_fails_policy:
AuthenticationErrorAllowedFails: 1 # cooldown after 2nd auth failure
TimeoutErrorAllowedFails: 3 # cooldown after 4th timeout
```
When `allowed_fails_policy` is set, the binary health check filter is bypassed. Only the cooldown system controls routing exclusion, and it only fires after your configured threshold is crossed.
### Step 4 (optional): Ignore transient errors
429 (rate limit) and 408 (timeout) from a health check usually mean the deployment is temporarily overloaded, not broken. To prevent these from affecting routing at all:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true # 429 and 408 never affect routing
```
With this on, only hard failures (401, 404, 5xx) from health checks contribute to cooldown.
## Full example
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY_SECONDARY
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true
router_settings:
cooldown_time: 60
allowed_fails_policy:
AuthenticationErrorAllowedFails: 0 # cooldown immediately on auth failure
TimeoutErrorAllowedFails: 2 # cooldown after 3 timeouts
RateLimitErrorAllowedFails: 5 # cooldown after 6 rate limits (if not ignoring transients)
```
## Configuration reference
| Setting | Where | Default | Description |
|---|---|---|---|
| `enable_health_check_routing` | `general_settings` | `false` | Route away from deployments that fail health checks |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between full health check cycles |
| `health_check_staleness_threshold` | `general_settings` | `interval x 2` | Seconds before cached health state is ignored |
| `health_check_ignore_transient_errors` | `general_settings` | `false` | Ignore 429 and 408 from health checks; these never affect routing |
| `cooldown_time` | `router_settings` | `5` | Seconds a deployment stays in cooldown after threshold is crossed |
| `allowed_fails_policy` | `router_settings` | `null` | Per-error-type failure thresholds before cooldown (see below) |
### `allowed_fails_policy` fields
| Field | Error type | HTTP status |
|---|---|---|
| `AuthenticationErrorAllowedFails` | Bad API key | 401 |
| `TimeoutErrorAllowedFails` | Request timeout | 408 |
| `RateLimitErrorAllowedFails` | Rate limit exceeded | 429 |
| `BadRequestErrorAllowedFails` | Malformed request | 400 |
| `ContentPolicyViolationErrorAllowedFails` | Content filtered | 400 |
The value is the number of failures **tolerated** before cooldown. `0` means cooldown on the first failure. `2` means cooldown on the third.
## Things to keep in mind
- **Counter TTL must be longer than the health check interval.** `allowed_fails_policy` works by incrementing a `failed_calls` counter per deployment. That counter expires after `cooldown_time` seconds. If `cooldown_time` is shorter than `health_check_interval`, the counter resets between every check cycle and failures never accumulate. Set `cooldown_time` greater than `health_check_interval` when using `allowed_fails_policy`.
```yaml
router_settings:
cooldown_time: 60 # must be > health_check_interval (30s here)
general_settings:
health_check_interval: 30
```
- **`AllowedFails: N` means cooldown on the (N+1)th failure.** The counter check is `updated_fails > allowed_fails`, so `0` triggers on the 1st failure, `1` on the 2nd, `2` on the 3rd.
| `AllowedFails` | Cooldown triggers after |
|---|---|
| `0` | 1st failure |
| `1` | 2nd failure |
| `2` | 3rd failure |
- **Without `allowed_fails_policy`, the first failure is enough.** The first failed health check immediately excludes the deployment from routing. Use `allowed_fails_policy` when you want tolerance for flaky checks.
- **If all deployments are unhealthy, the filter is bypassed.** Traffic keeps flowing rather than returning no deployment at all. Requests will fail, but the router keeps trying.
- **Health check failures and request failures share the same counters.** When `allowed_fails_policy` is set, both sources increment the same `failed_calls` counter. A deployment at 1 health check failure that then receives 1 failing request will hit the threshold for `AllowedFails: 1` and enter cooldown.
## Debugging
Run the proxy with `--detailed_debug` and look for these log lines:
After each health check cycle (written at DEBUG level):
```
health_check_routing_state_updated healthy=2 unhealthy=1
```
When a health check failure increments the counter and triggers cooldown (DEBUG level):
```
checks 'should_run_cooldown_logic'
Attempting to add <deployment_id> to cooldown list
```
When safety net fires because all deployments are in cooldown:
```
All deployments in cooldown via health-check routing, bypassing cooldown filter
```
When safety net fires because all deployments are unhealthy (binary filter, no `allowed_fails_policy`):
```
All deployments marked unhealthy by health checks, bypassing health filter
```
docs/my-website/docs/proxy/high_availability_control_plane.md
+2 -2
View File
@@ -42,9 +42,9 @@ The **High Availability Control Plane** takes a different approach:
<ControlPlaneArchitecture />
The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration.
The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It is **not a router** — it does not proxy or route any LLM requests. It exists purely so admins can switch between workers and manage them from a single UI.
Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets.
Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own database, Redis, users, keys, teams, and budgets. No infrastructure is shared between workers.
## Setup
docs/my-website/docs/proxy/jwt_key_mapping.md
+318
View File
@@ -0,0 +1,318 @@
# JWT → Virtual Key Mapping
:::info Enterprise
JWT → Virtual Key Mapping is an Enterprise feature.
[Get a free trial](https://enterprise.litellm.ai/demo)
:::
Map JWT tokens to LiteLLM virtual keys — so every JWT client gets the same granular controls as a virtual key: model restrictions, spend limits, rate limits, guardrails, and full spend tracking.
**Why this matters:** Standard JWT auth maps a JWT to a *team*. That's a shared boundary — all clients under a team share the same limits. With JWT → Virtual Key Mapping, each individual JWT client (identified by a claim like `client_id`, `azp`, or `sub`) maps to its own virtual key. You get per-client accountability without issuing API keys to your users.
**Common use case:** Your company uses SSO/OIDC. Developers use Claude Code with their identity tokens. You want to enforce per-developer model access and spend limits without giving each person a LiteLLM API key.
---
## How It Works
```mermaid
sequenceDiagram
participant Client as Client (Claude Code / API)
participant Proxy as LiteLLM Proxy
participant OIDC as OIDC Provider
participant DB as Mapping Table
Client->>Proxy: POST /v1/chat/completions<br/>Authorization: Bearer <JWT>
Proxy->>OIDC: Verify JWT signature
OIDC-->>Proxy: Valid ✓
Proxy->>Proxy: Extract claim<br/>(e.g. client_id = "alice@corp.com")
Proxy->>DB: Look up (claim_name, claim_value)
alt Mapping found
DB-->>Proxy: virtual_key_id = sk-abc123
Proxy->>Proxy: Apply virtual key permissions<br/>(models, budget, rate limits)
Proxy-->>Client: 200 OK
else No mapping — fallback_team_mapping
Proxy->>Proxy: Fall through to team JWT auth
Proxy-->>Client: 200 OK
else No mapping — reject
Proxy-->>Client: 403 Forbidden
else No mapping — auto_register
Proxy->>DB: Create new virtual key + mapping
Proxy-->>Client: 200 OK
end
```
---
## Setup
### Prerequisites
Complete [OIDC JWT Auth setup](./token_auth.md) first — you need `JWT_PUBLIC_KEY_URL` configured and `enable_jwt_auth: True` in your proxy config.
### Step 1. Configure the JWT claim to map on
Add `jwt_client_id_field` to your `litellm_jwtauth` config. This is the JWT claim LiteLLM uses as the lookup key:
```yaml
general_settings:
master_key: sk-1234
enable_jwt_auth: True
litellm_jwtauth:
team_id_jwt_field: "team_id" # existing team mapping (optional)
user_id_jwt_field: "sub"
jwt_client_id_field: "client_id" # 👈 claim used for key mapping
unregistered_jwt_client_behavior: "fallback_team_mapping" # see below
```
**`unregistered_jwt_client_behavior`** controls what happens when a JWT has no registered mapping:
| Value | Behavior |
|-------|----------|
| `fallback_team_mapping` | Fall through to team-based JWT auth (default — backward compatible) |
| `reject` | Return 403 if no mapping found |
| `auto_register` | Auto-create a virtual key + mapping on first encounter |
### Step 2. Register a JWT client → virtual key mapping
**Option A: Single call (creates key + mapping atomically)**
```bash
curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "dev-alice",
"models": ["claude-sonnet-4-5", "claude-haiku-4-5"],
"max_budget": 50.0,
"budget_duration": "30d",
"rpm_limit": 100,
"tpm_limit": 50000,
"team_id": "engineering"
}'
```
Response includes the virtual key token (only shown on creation):
```json
{
"key": "sk-abc123...",
"key_id": "key_123",
"mapping_id": "mapping_456",
"jwt_claim_name": "client_id",
"jwt_claim_value": "dev-alice"
}
```
**Option B: Map an existing virtual key**
```bash
curl -X POST 'http://0.0.0.0:4000/jwt/key/mapping/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "dev-alice",
"virtual_key_id": "key_123"
}'
```
### Step 3. Test it
```bash
# Get a JWT from your OIDC provider (must have client_id: dev-alice)
JWT_TOKEN="eyJhbG..."
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H "Authorization: Bearer $JWT_TOKEN" \
-H 'Content-Type: application/json' \
-d '{
"model": "claude-sonnet-4-5",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
The request is now tracked against `dev-alice`'s virtual key — spend, rate limits, and model access enforced per-client.
---
## Walkthrough: Admin grants granular access, team uses Claude Code
This is the full flow for an engineering team using Claude Code with company SSO.
### Admin setup
**1. Create a team for engineering**
```bash
curl -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"team_alias": "engineering",
"models": ["claude-sonnet-4-5", "claude-haiku-4-5"]
}'
```
**2. Register each developer with their own key and spend limit**
```bash
# Alice — senior eng, higher budget
curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "alice@corp.com",
"team_id": "engineering",
"models": ["claude-sonnet-4-5", "claude-haiku-4-5"],
"max_budget": 200.0,
"budget_duration": "30d",
"rpm_limit": 200
}'
# Bob — contractor, tighter limits
curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "bob@contractor.com",
"team_id": "engineering",
"models": ["claude-haiku-4-5"],
"max_budget": 20.0,
"budget_duration": "30d",
"rpm_limit": 30
}'
```
**3. Configure Claude Code to use the proxy**
Set the proxy as the API base in your team's Claude Code config:
```bash
# Point Claude Code at the LiteLLM proxy instead of Anthropic directly.
# ANTHROPIC_API_KEY here is the bearer token sent to the proxy — set it to
# the user's SSO/OIDC JWT token (obtained from your IdP at login).
export ANTHROPIC_API_KEY="<user-sso-jwt-token>"
export ANTHROPIC_BASE_URL="http://your-litellm-proxy:4000"
```
Or in `~/.claude/settings.json`:
```json
{
"env": {
"ANTHROPIC_BASE_URL": "http://your-litellm-proxy:4000"
}
}
```
**4. Developers authenticate with SSO as usual**
When Alice runs Claude Code, her JWT (issued by your IdP with `client_id: alice@corp.com`) goes to the proxy. LiteLLM looks up the mapping, finds her virtual key, and enforces her specific limits — her $200/month budget, 200 RPM cap, and access to Sonnet and Haiku only.
Bob's token maps to his own key — $20/month, Haiku only, 30 RPM.
No API keys distributed. No shared limits. Full per-developer spend visibility in the LiteLLM dashboard.
---
## Managing mappings
**View a mapping + its key settings**
```bash
curl 'http://0.0.0.0:4000/jwt/key/mapping/info?jwt_claim_name=client_id&jwt_claim_value=alice@corp.com' \
-H 'Authorization: Bearer <MASTER_KEY>'
```
Response includes the linked key's `models`, `max_budget`, `spend`, `rpm_limit`, `expires`, etc.
**Update a mapping**
```bash
curl -X POST 'http://0.0.0.0:4000/jwt_client/update' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "alice@corp.com",
"max_budget": 300.0
}'
```
**Delete a mapping**
```bash
curl -X DELETE 'http://0.0.0.0:4000/jwt/key/mapping/delete' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "client_id",
"jwt_claim_value": "alice@corp.com"
}'
```
---
## Security
JWT-bound keys are locked down:
- Non-admin users cannot call `/key/update`, `/key/delete`, or `/key/regenerate` on a JWT-bound key. These return 403.
- JWT-bound keys are automatically restricted to `llm_api_routes` — they can make LLM calls but cannot manage other keys or admin resources.
- Only proxy admins can create, update, or delete mappings.
---
## Multi-IdP support
If you have users across multiple identity providers that share the same claim values (e.g. two services both have `sub: user-123` from different issuers), set `issuer` when creating the mapping:
```bash
curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"jwt_claim_name": "sub",
"jwt_claim_value": "user-123",
"issuer": "https://idp-a.corp.com",
"models": ["claude-sonnet-4-5"],
"max_budget": 50.0
}'
```
Mappings are unique per `(claim_name, claim_value, issuer)` — so `user-123` from IdP A and `user-123` from IdP B resolve to different virtual keys.
---
## What JWT clients can and can't do vs virtual keys
| Capability | Virtual Key | JWT → Key Mapping |
|---|---|---|
| Per-client model access | ✅ | ✅ |
| Per-client spend budget | ✅ | ✅ |
| Per-client RPM/TPM limits | ✅ | ✅ |
| Team membership | ✅ | ✅ |
| Spend tracking in dashboard | ✅ | ✅ |
| Guardrails | ✅ | ✅ |
| Key rotation | ✅ | ✅ (admin only) |
| Key expiry | ✅ | ✅ |
| No API key distribution needed | ❌ | ✅ |
| Works with existing SSO/OIDC | ❌ | ✅ |
---
## Related
- [OIDC JWT Auth](./token_auth.md) — base JWT auth setup required before using this feature
- [Virtual Keys](./virtual_keys.md) — full virtual key documentation
- [Access Control](./access_control.md) — model and team access control
docs/my-website/docs/proxy/oauth2.md
+21
View File
@@ -61,3 +61,24 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug)
## Using OAuth2 + JWT Together
If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths:
- JWT validation for user tokens
- OAuth2 introspection for machine tokens
For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2).
docs/my-website/docs/proxy/self_serve.md
+17 -8
View File
@@ -358,10 +358,15 @@ When you connect litellm to your SSO provider, litellm can auto-create teams. Us
```yaml showLineNumbers title="Default Params for new teams"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
- "/key/generate" # Allow members to generate API keys
```
@@ -390,10 +395,14 @@ litellm_settings:
max_budget_in_team: 100 # Optional[float], optional): $100 budget for the team. Defaults to None.
user_role: "user" # Optional[str], optional): "user" or "admin". Defaults to "user"
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity"
upperbound_key_generate_params: # Upperbound for /key/generate requests when self-serve flow is on
docs/my-website/docs/proxy/team_model_add.md
+1 -1
View File
@@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \
"model": "openai/gpt-4o",
"custom_llm_provider": "openai",
"api_key": "******ccb07",
"api_base": "https://my-endpoint-sweden-berri992.openai.azure.com",
"api_base": "https://my-azure-endpoint.openai.azure.com",
"api_version": "2023-12-01-preview"
},
"model_info": {
docs/my-website/docs/proxy/token_auth.md
+47
View File
@@ -16,6 +16,12 @@ Use JWT's to auth admins / users / projects into the proxy.
:::
:::tip JWT → Virtual Key Mapping
Want per-user model restrictions, spend limits, and rate limits without distributing API keys? See **[JWT → Virtual Key Mapping](./jwt_key_mapping.md)** — enterprise-grade granular access control for JWT-authenticated users (e.g. Claude Code + SSO).
:::
## Usage
### Step 1. Setup Proxy
@@ -784,6 +790,47 @@ litellm_jwtauth:
user_roles_jwt_field: "resource_access.your-client.roles"
```
## Route JWT-Shaped Machine Tokens to OAuth2
Use this when both are enabled:
- `enable_jwt_auth: true` for standard JWT validation
- `enable_oauth2_auth: true` for OAuth2 introspection
If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2.
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
user_id_jwt_field: "sub"
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
### Matching behavior
- A rule matches when all configured selectors match token claims
- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional)
- Selector values support both string and list forms
- If no rule matches, LiteLLM continues with standard JWT validation
### List-based override example
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: ["machine-issuer.example.com", "backup-issuer.example.com"]
client_id: ["MID_LITELLM", "MID_BACKUP"]
aud: ["api://litellm", "api://fallback"]
path: "oauth2"
```
## [BETA] Control Access with OIDC Roles
Allow JWT tokens with supported roles to access the proxy.
docs/my-website/docs/realtime.md
+1 -1
View File
@@ -82,7 +82,7 @@ Run this script using node - `node test.js`
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `sk-1234`,
docs/my-website/docs/tutorials/claude_responses_api.md
+1 -1
View File
@@ -214,7 +214,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
docs/my-website/docs/tutorials/github_copilot_integration.md
+1 -1
View File
@@ -141,7 +141,7 @@ Route requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
docs/my-website/docs/tutorials/google_genai_sdk.md
+1 -1
View File
@@ -282,7 +282,7 @@ Route `gemini-2.5-flash` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
docs/my-website/docs/tutorials/litellm_gemini_cli.md
+2 -2
View File
@@ -127,7 +127,7 @@ Route `gemini-2.5-pro` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@@ -149,7 +149,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
docs/my-website/docs/tutorials/litellm_qwen_code_cli.md
+2 -2
View File
@@ -129,7 +129,7 @@ Route `qwen-code` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@@ -151,7 +151,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
docs/my-website/docs/tutorials/msft_sso.md
+6 -4
View File
@@ -123,10 +123,12 @@ Navigate to your litellm config file and set the following params
```yaml showLineNumbers title="litellm config with default_team_params"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
```
### 3.2 Auto-create a new team on LiteLLM
docs/my-website/docusaurus.config.js
+1
View File
@@ -284,6 +284,7 @@ const config = {
label: 'Enterprise',
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Changelog', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{
href: 'https://github.com/BerriAI/litellm',
docs/my-website/img/april_townhall_banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

docs/my-website/package-lock.json
Generated
+9 -16
View File
@@ -13,18 +13,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"engines": {
"node": ">=16.14",
@@ -20403,13 +20403,6 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",
docs/my-website/package.json
+28 -41
View File
@@ -19,18 +19,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"browserslist": {
"production": [
@@ -48,27 +48,26 @@
"node": ">=16.14",
"npm": ">=8.3.0"
},
"resolutions": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"node-forge": ">=1.3.2"
},
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
"serialize-javascript": ">=7.0.3",
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1",
"lodash-es": ">=4.17.23",
"webpack-dev-server": "5.2.3",
"form-data": "4.0.5",
"mermaid": "11.12.1",
"minimatch": "10.2.4",
"serialize-javascript": "7.0.3",
"mdast-util-to-hast": "13.2.1",
"lodash-es": "4.17.23",
"@babel/traverse": "7.28.5",
"ws": "8.19.0",
"http-proxy-middleware": "3.0.5",
"tar-fs": "3.1.1",
"webpack-dev-middleware": "5.3.4",
"braces": "3.0.3",
"webpack": "5.105.3",
"serve-static": "2.2.1",
"path-to-regexp": "1.9.0",
"dompurify": "3.3.2",
"svgo": "4.0.1",
"schema-utils@3": {
"ajv": "6.14.0"
},
@@ -83,18 +82,6 @@
},
"url-loader": {
"ajv": "6.14.0"
},
"@babel/traverse": ">=7.23.2",
"ws": ">=7.5.10",
"http-proxy-middleware": ">=2.0.9",
"tar-fs": ">=2.1.4",
"webpack-dev-middleware": ">=5.3.4",
"braces": ">=3.0.3",
"axios": "1.13.6",
"webpack": ">=5.94.0",
"serve-static": ">=1.16.0",
"path-to-regexp": ">=0.1.12",
"dompurify": ">=3.3.2",
"svgo": ">=3.3.3"
}
}
}
docs/my-website/release_notes/v1.83.0/index.md
+62
View File
@@ -0,0 +1,62 @@
---
title: "v1.83.0 - Official Release (Post Supply Chain Incident)"
slug: "v1-83-0"
date: 2026-03-31T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-1.83.0-nightly
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.83.0
```
</TabItem>
</Tabs>
## Context: First Release After Supply Chain Incident
v1.83.0 is the first LiteLLM release built and published through our new [CI/CD v2 pipeline](https://docs.litellm.ai/blog/ci-cd-v2-improvements), following the [supply chain incident on March 24](https://docs.litellm.ai/blog/security-update-march-2026).
We paused all releases for one week while we:
1. Completed a forensic review with [Mandiant](https://www.mandiant.com/) and [Veria Labs](https://verialabs.com/)
2. Rebuilt the release pipeline from scratch with isolated environments and ephemeral credentials
3. Verified the codebase contains no indicators of compromise
If you have questions about this release or the incident, see our [Security Townhall post](https://docs.litellm.ai/blog/security-townhall-updates) or reach out at `security@berri.ai`.
---
## Links
- **PyPI**: [litellm 1.83.0](https://pypi.org/project/litellm/1.83.0/)
- **Security update**: [Supply chain incident report](https://docs.litellm.ai/blog/security-update-march-2026)
- **Security townhall**: [What happened, what we've done, what comes next](https://docs.litellm.ai/blog/security-townhall-updates)
- **CI/CD v2**: [Announcing CI/CD v2 for LiteLLM](https://docs.litellm.ai/blog/ci-cd-v2-improvements)
- **April stability sprint**: [Help us plan](https://github.com/BerriAI/litellm/issues/24825)
docs/my-website/sidebars.js
+3 -1
View File
@@ -452,6 +452,7 @@ const sidebars = {
items: [
"proxy/virtual_keys",
"proxy/token_auth",
"proxy/jwt_key_mapping",
"proxy/service_accounts",
"proxy/access_control",
"proxy/cli_sso",
@@ -1050,7 +1051,8 @@ const sidebars = {
"proxy/fallback_management",
"proxy/tag_routing",
"proxy/timeout",
"wildcard_routing"
"wildcard_routing",
"proxy/health_check_routing"
],
},
{
docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx
+69 -44
View File
@@ -3,23 +3,56 @@ import styles from './styles.module.css';
/* ────────────────────── Shared small pieces ────────────────────── */
function InfraChip({ color, label }: { color: string; label: string }) {
const dotClass =
function InfraBox({ icon, label, color }: { icon: string; label: string; color: 'green' | 'blue' | 'orange' }) {
const colorClass =
color === 'green'
? styles.infraDotGreen
? styles.infraBoxGreen
: color === 'blue'
? styles.infraDotBlue
: styles.infraDotOrange;
? styles.infraBoxBlue
: styles.infraBoxOrange;
return (
<span className={styles.infraChip}>
<span className={`${styles.infraDot} ${dotClass}`} />
{label}
</span>
<div className={`${styles.infraBox} ${colorClass}`}>
<span className={styles.infraBoxIcon}>{icon}</span>
<span className={styles.infraBoxLabel}>{label}</span>
</div>
);
}
/* ────────────────────── Architecture tab ────────────────────── */
/* ────────────────────── Worker column with infra ────────────────────── */
function WorkerColumn({
name,
region,
subtitle,
nodeClass,
badgeClass,
}: {
name: string;
region: string;
subtitle: string;
nodeClass: string;
badgeClass: string;
}) {
return (
<div className={styles.workerColumn}>
<div className={`${styles.node} ${styles.nodeWorker} ${nodeClass}`}>
<div className={styles.nodeHeader}>
<span className={styles.nodeTitle}>{name}</span>
<span className={`${styles.badge} ${badgeClass}`}>{region}</span>
</div>
<div className={styles.nodeSubtitle}>{subtitle}</div>
<div className={styles.nodeCaption}>Handles LLM requests</div>
</div>
<div className={styles.infraStack}>
<InfraBox icon="🗄" label="Own Database" color="green" />
<InfraBox icon="⚡" label="Own Redis" color="orange" />
</div>
</div>
);
}
/* ────────────────────── Architecture diagram ────────────────────── */
function ArchitectureView() {
return (
@@ -36,49 +69,41 @@ function ArchitectureView() {
<div className={`${styles.node} ${styles.nodeControlPlane}`}>
<div className={styles.nodeHeader}>
<span className={styles.nodeTitle}>Control Plane</span>
<span className={`${styles.badge} ${styles.badgeBlue}`}>UI</span>
<span className={`${styles.badge} ${styles.badgeBlue}`}>ADMIN UI ONLY</span>
</div>
<div className={styles.nodeSubtitle}>cp.example.com</div>
<div className={styles.infraRow}>
<InfraChip color="green" label="Own DB" />
<InfraChip color="orange" label="Own Redis" />
<InfraChip color="blue" label="Own Key" />
<div className={styles.nodeCaption}>
Not a router — does not proxy LLM requests.
<br />
Lets admins switch between workers to manage them.
</div>
</div>
{/* Branch connector */}
<div className={styles.connectorBranch}>
<div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
<div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
{/* Branch connector with label */}
<div className={styles.connectorBranchLabeled}>
<span className={styles.connectorLabel}>UI management only</span>
<div className={styles.connectorBranch}>
<div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
<div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
</div>
</div>
{/* Workers */}
<div className={styles.workersRow}>
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerA}`}>
<div className={styles.nodeHeader}>
<span className={styles.nodeTitle}>Worker A</span>
<span className={`${styles.badge} ${styles.badgeGreen}`}>US East</span>
</div>
<div className={styles.nodeSubtitle}>worker-a.example.com</div>
<div className={styles.infraRow}>
<InfraChip color="green" label="Own DB" />
<InfraChip color="orange" label="Own Redis" />
<InfraChip color="blue" label="Own Key" />
</div>
</div>
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerB}`}>
<div className={styles.nodeHeader}>
<span className={styles.nodeTitle}>Worker B</span>
<span className={`${styles.badge} ${styles.badgePurple}`}>EU West</span>
</div>
<div className={styles.nodeSubtitle}>worker-b.example.com</div>
<div className={styles.infraRow}>
<InfraChip color="green" label="Own DB" />
<InfraChip color="orange" label="Own Redis" />
<InfraChip color="blue" label="Own Key" />
</div>
</div>
<WorkerColumn
name="Worker A"
region="US East"
subtitle="worker-a.example.com"
nodeClass={styles.nodeWorkerA}
badgeClass={styles.badgeGreen}
/>
<WorkerColumn
name="Worker B"
region="EU West"
subtitle="worker-b.example.com"
nodeClass={styles.nodeWorkerB}
badgeClass={styles.badgePurple}
/>
</div>
</div>
);
docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css
+78 -28
View File
@@ -284,45 +284,67 @@
color: var(--cp-purple);
}
/* ── Infrastructure chips ── */
.infraRow {
display: flex;
gap: 0.4rem;
justify-content: center;
flex-wrap: wrap;
margin-top: 0.5rem;
/* ── Node caption ── */
.nodeCaption {
font-size: 0.72rem;
color: var(--cp-text-muted);
margin-top: 0.4rem;
line-height: 1.4;
font-style: italic;
}
.infraChip {
/* ── Infrastructure boxes (per-worker) ── */
.infraStack {
display: flex;
flex-direction: column;
gap: 0.35rem;
margin-top: 0.5rem;
width: 100%;
}
.infraBox {
display: flex;
align-items: center;
gap: 0.3rem;
font-size: 0.7rem;
font-weight: 500;
color: var(--cp-text-secondary);
background: var(--cp-infra-bg);
border: 1px solid var(--cp-infra-border);
border-radius: 6px;
padding: 0.2rem 0.5rem;
gap: 0.5rem;
padding: 0.45rem 0.75rem;
border-radius: 8px;
border: 1.5px solid var(--cp-border);
background: var(--cp-card-bg);
}
.infraDot {
width: 6px;
height: 6px;
border-radius: 50%;
.infraBoxGreen {
border-color: var(--cp-green);
background: var(--cp-green-light);
}
.infraBoxBlue {
border-color: var(--cp-accent);
background: var(--cp-accent-light);
}
.infraBoxOrange {
border-color: var(--cp-orange);
background: var(--cp-orange-light);
}
.infraBoxIcon {
font-size: 0.85rem;
flex-shrink: 0;
}
.infraDotGreen {
background: var(--cp-green);
.infraBoxLabel {
font-size: 0.75rem;
font-weight: 600;
color: var(--cp-text);
}
.infraDotBlue {
background: var(--cp-accent);
}
.infraDotOrange {
background: var(--cp-orange);
/* ── Worker column (card + infra stack) ── */
.workerColumn {
display: flex;
flex-direction: column;
align-items: stretch;
min-width: 220px;
max-width: 260px;
}
/* ── Workers row ── */
@@ -333,6 +355,24 @@
flex-wrap: wrap;
}
/* ── Connector with label ── */
.connectorBranchLabeled {
display: flex;
flex-direction: column;
align-items: center;
width: 100%;
max-width: 700px;
}
.connectorLabel {
font-size: 0.7rem;
color: var(--cp-text-muted);
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 0.25rem;
}
/* ── Animated flow ── */
.flowLabel {
font-size: 0.7rem;
@@ -511,6 +551,16 @@
max-width: 260px;
}
.workerColumn {
min-width: auto;
width: 100%;
max-width: 280px;
}
.connectorBranchLabeled {
display: none;
}
.connectorBranch {
display: none;
}
litellm-js/proxy/.npmrc
+5
View File
@@ -0,0 +1,5 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
litellm-js/proxy/package.json
+4 -4
View File
@@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "^4.1.4",
"openai": "^4.29.2"
"hono": "4.1.4",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20240208.0",
"wrangler": "^3.32.0"
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
}
}
litellm-js/spend-logs/.npmrc
+5
View File
@@ -0,0 +1,5 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
litellm-js/spend-logs/Dockerfile
+1 -1
View File
@@ -8,7 +8,7 @@ WORKDIR /app
COPY ./litellm-js/spend-logs/package*.json ./
# Install dependencies
RUN npm install
RUN npm ci
# Install Prisma globally
RUN npm install -g prisma
litellm-js/spend-logs/package-lock.json
Generated
+4 -4
View File
@@ -5,12 +5,12 @@
"packages": {
"": {
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {
litellm-js/spend-logs/package.json
+5 -22
View File
@@ -3,28 +3,11 @@
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
},
"overrides": {
"glob": ">=11.1.0",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
"@babel/traverse": ">=7.23.2",
"ws": ">=7.5.10",
"http-proxy-middleware": ">=2.0.9",
"tar-fs": ">=2.1.4",
"webpack-dev-middleware": ">=5.3.4",
"braces": ">=3.0.3",
"axios": "1.13.6",
"webpack": ">=5.94.0",
"serve-static": ">=1.16.0",
"path-to-regexp": ">=0.1.12"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}
}
litellm-proxy-extras/litellm_proxy_extras/migrations/20260319000000_restore_mcp_approval_fields/migration.sql
+13
View File
@@ -0,0 +1,13 @@
-- Restore fields dropped by 20260311180521_schema_sync on LiteLLM_MCPServerTable
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable"
ADD COLUMN IF NOT EXISTS "source_url" TEXT,
ADD COLUMN IF NOT EXISTS "approval_status" TEXT DEFAULT 'active',
ADD COLUMN IF NOT EXISTS "submitted_by" TEXT,
ADD COLUMN IF NOT EXISTS "submitted_at" TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS "reviewed_at" TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS "review_notes" TEXT;
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_MCPServerTable_approval_status_idx"
ON "LiteLLM_MCPServerTable"("approval_status");
litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql
+12
View File
@@ -0,0 +1,12 @@
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development';
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT;
-- DropIndex (old unique constraint)
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key";
-- CreateIndex (new unique constraint)
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment");
-- CreateIndex (new composite index)
CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment");
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+4 -1
View File
@@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.62"
version = "0.4.64"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.62"
version = "0.4.64"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+1
View File
@@ -1838,6 +1838,7 @@ if TYPE_CHECKING:
)
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
from .llms.lambda_ai.chat.transformation import (
litellm/_logging.py
+35 -3
View File
@@ -26,6 +26,12 @@ _REDACTED = "REDACTED"
def _build_secret_patterns() -> re.Pattern:
patterns: List[str] = [
# ── PEM private key / certificate blocks ──
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# ── GCP OAuth2 access tokens (ya29.*) ──
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# ── Credential %s formatting (space separator, no key= prefix) ──
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
@@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern:
# Google API keys
r"AIza[0-9A-Za-z\-_]{35}",
# Password / secret params (handles key=value and 'key': 'value')
r"\w*(?:password|passwd|client_secret|secret_key|_secret)"
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
@@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern:
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"private_key|signing_key|encryption_key|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# ── Raw JWTs (without Bearer prefix) ──
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# ── Azure SAS tokens in URLs ──
r"[?&]sig=[A-Za-z0-9%+/=]+",
# ── Full JSON service-account blobs (single-line and multi-line) ──
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
@@ -74,6 +89,23 @@ def _redact_string(value: str) -> str:
return _SECRET_RE.sub(_REDACTED, value)
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
Use this for code paths that bypass the logging system — e.g. Slack/Teams
alerting, HTTP error response bodies, or any other string that may contain
secrets and will be sent to an external sink.
Not to be confused with redact_message_input_output_from_logging() in
litellm_core_utils/redact_messages.py, which redacts LLM prompt/response
content for privacy — this function redacts credential patterns (API keys,
PEM blocks, tokens, etc.) by shape.
"""
if not _ENABLE_SECRET_REDACTION:
return value
return _redact_string(value)
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@@ -441,7 +473,7 @@ def _enable_debugging():
def print_verbose(print_statement):
try:
if set_verbose:
print(print_statement) # noqa
print(redact_secrets(str(print_statement))) # noqa
except Exception:
pass
litellm/a2a_protocol/litellm_completion_bridge/handler.py
+6 -8
View File
@@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
)
return response_data
@@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
@@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler:
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk
litellm/a2a_protocol/providers/base.py
+3 -3
View File
@@ -3,7 +3,7 @@ Base configuration for A2A protocol providers.
"""
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
class BaseA2AProviderConfig(ABC):
@@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
@@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""
litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py
+22
View File
@@ -0,0 +1,22 @@
"""
Bedrock AgentCore A2A provider.
Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively,
bypassing the completion bridge that would otherwise strip the envelope.
"""
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
__all__ = [
"BedrockAgentCoreA2AConfig",
"BedrockAgentCoreA2AHandler",
"BedrockAgentCoreA2ATransformation",
]
litellm/a2a_protocol/providers/bedrock_agentcore/config.py
+61
View File
@@ -0,0 +1,61 @@
"""
Bedrock AgentCore A2A provider configuration.
"""
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig):
"""
Provider configuration for Bedrock AgentCore A2A-native agents.
AgentCore agents that speak A2A natively expect the full JSON-RPC envelope.
This config bypasses the completion bridge and forwards requests directly,
deriving the endpoint URL from the model ARN and signing with SigV4/JWT.
"""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
return await BedrockAgentCoreA2AHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
async for chunk in BedrockAgentCoreA2AHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
):
yield chunk
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py
+134
View File
@@ -0,0 +1,134 @@
"""
Handler for Bedrock AgentCore A2A-native agents.
Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the
completion bridge that would otherwise strip the envelope.
"""
import json
from typing import Any, AsyncIterator, Dict, cast
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
class BedrockAgentCoreA2AHandler:
"""
Handler for Bedrock AgentCore A2A requests.
Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig,
and POSTs directly to the AgentCore endpoint.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Returns:
A2A JSON-RPC response dict from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending non-streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
)
response.raise_for_status()
response_data = response.json()
if "error" in response_data:
verbose_logger.warning(
f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}"
)
return response_data
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Yields:
A2A streaming response events from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
stream=True,
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
stream=True,
)
response.raise_for_status()
# Check content type — AgentCore may return JSON instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
# Single JSON response fallback (not SSE)
verbose_logger.debug(
"BedrockAgentCore A2A streaming: received JSON instead of SSE, "
"yielding as single event"
)
response_body = await response.aread()
response_data = json.loads(response_body)
yield response_data
else:
# SSE stream — parse data: lines
async for event in BedrockAgentCoreA2ATransformation.parse_sse_events(
response
):
yield event
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
+134
View File
@@ -0,0 +1,134 @@
"""
Transformation layer for Bedrock AgentCore A2A provider.
Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs,
and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
"""
import json
from typing import Any, AsyncIterator, Dict, Tuple
from litellm._logging import verbose_logger
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
Reuses AmazonAgentCoreConfig for URL construction, ARN parsing,
and request signing. No logic is duplicated.
"""
@staticmethod
def get_url_and_signed_request(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
method: str = "message/send",
stream: bool = False,
) -> Tuple[str, dict, bytes]:
"""
Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams
litellm_params: Agent's litellm_params (model, api_key, etc.)
method: JSON-RPC method name (default: "message/send")
stream: Whether this is a streaming request
Returns:
Tuple of (url, signed_headers, signed_body_bytes)
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
agentcore_config = AmazonAgentCoreConfig()
# Derive URL from ARN
url = agentcore_config.get_complete_url(
api_base=optional_params.get("api_base"),
api_key=optional_params.get("api_key"),
model=agentcore_model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)
# Construct JSON-RPC 2.0 envelope
json_rpc_body = {
"jsonrpc": "2.0",
"method": method,
"id": request_id,
"params": params,
}
# Set required AgentCore session headers (normally set by transform_request,
# which we skip because it also builds {"prompt": "..."})
headers: dict = {}
session_id = agentcore_config._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id = agentcore_config._get_runtime_user_id(optional_params)
if runtime_user_id:
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
# Sign the request (SigV4 or JWT depending on api_key presence)
signed_headers, signed_body = agentcore_config.sign_request(
headers=headers,
optional_params=optional_params,
request_data=json_rpc_body,
api_base=url,
api_key=optional_params.get("api_key"),
model=agentcore_model,
stream=stream,
)
# sign_request returns Optional[bytes] — ensure we have bytes
if signed_body is None:
signed_body = json.dumps(json_rpc_body).encode()
return url, signed_headers, signed_body
@staticmethod
async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]:
"""
Parse SSE events from an httpx streaming response.
Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict.
Args:
response: httpx streaming response
Yields:
Parsed JSON dicts from SSE data lines
"""
async for line in response.aiter_lines():
line = line.strip()
if not line:
continue
if line.startswith("data:"):
data_str = line[len("data:") :].strip()
if not data_str:
continue
try:
event = json.loads(data_str)
yield event
except json.JSONDecodeError:
verbose_logger.debug(
f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}"
)
continue
litellm/a2a_protocol/providers/config_manager.py
+8 -4
View File
@@ -19,12 +19,14 @@ class A2AProviderConfigManager:
@staticmethod
def get_provider_config(
custom_llm_provider: Optional[str],
model: Optional[str] = None,
) -> Optional[BaseA2AProviderConfig]:
"""
Get the provider configuration for a given custom_llm_provider.
Args:
custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents")
model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock)
Returns:
Provider configuration instance or None if not found
@@ -39,9 +41,11 @@ class A2AProviderConfigManager:
return PydanticAIProviderConfig()
# Add more providers here as needed
# elif custom_llm_provider == "another_provider":
# from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig
# return AnotherProviderConfig()
if custom_llm_provider == "bedrock" and model and "agentcore" in model:
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
return BedrockAgentCoreA2AConfig()
return None
litellm/a2a_protocol/providers/pydantic_ai_agents/config.py
+7 -3
View File
@@ -2,7 +2,7 @@
Pydantic AI provider configuration.
"""
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
@@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to Pydantic AI agent."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
return await PydanticAIHandler.handle_non_streaming(
request_id=request_id,
params=params,
@@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request with fake streaming."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
async for chunk in PydanticAIHandler.handle_streaming(
request_id=request_id,
params=params,
litellm/constants.py
+4
View File
@@ -1319,6 +1319,9 @@ LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
LITELLM_KEY_ROTATION_GRACE_PERIOD: str = os.getenv(
"LITELLM_KEY_ROTATION_GRACE_PERIOD", ""
) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default)
LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int(
os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600)
) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
@@ -1347,6 +1350,7 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
)
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
litellm/cost_calculator.py
+2 -2
View File
@@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915
)
if (
model_info.get("input_cost_per_token", 0) > 0
or model_info.get("output_cost_per_token", 0) > 0
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
return generic_cost_per_token(
model=model,
litellm/experimental_mcp_client/client.py
+49 -1
View File
@@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth):
aws_session_token: Optional[str] = None,
aws_region_name: Optional[str] = None,
aws_service_name: Optional[str] = None,
aws_role_name: Optional[str] = None,
aws_session_name: Optional[str] = None,
):
try:
from botocore.credentials import Credentials
@@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth):
# Note: os.environ/ prefixed values are already resolved by
# ProxyConfig._check_for_os_environ_vars() at config load time.
# Values arrive here as plain strings.
if aws_access_key_id and aws_secret_access_key:
if aws_role_name:
self.credentials = self._assume_role(
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=self.region_name,
)
elif aws_access_key_id and aws_secret_access_key:
self.credentials = Credentials(
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
@@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth):
"(env vars, ~/.aws/credentials, instance profile)."
)
@staticmethod
def _assume_role(
aws_role_name: str,
aws_session_name: Optional[str],
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
aws_region_name: str,
):
"""Call STS AssumeRole and return temporary credentials."""
import boto3
from botocore.credentials import Credentials
session_name = (
aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
)
sts_kwargs: dict = {"region_name": aws_region_name}
if aws_access_key_id and aws_secret_access_key:
sts_kwargs["aws_access_key_id"] = aws_access_key_id
sts_kwargs["aws_secret_access_key"] = aws_secret_access_key
if aws_session_token:
sts_kwargs["aws_session_token"] = aws_session_token
sts_client = boto3.client("sts", **sts_kwargs)
sts_response = sts_client.assume_role(
RoleArn=aws_role_name,
RoleSessionName=session_name,
)
sts_creds = sts_response["Credentials"]
return Credentials(
access_key=sts_creds["AccessKeyId"],
secret_key=sts_creds["SecretAccessKey"],
token=sts_creds["SessionToken"],
)
def auth_flow(
self, request: httpx.Request
) -> Generator[httpx.Request, httpx.Response, None]:
litellm/integrations/azure_storage/azure_storage.py
+2 -3
View File
@@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
Gets Azure AD token to use for Azure Storage API requests
"""
verbose_logger.debug("Getting Azure AD Token from Azure Storage")
verbose_logger.debug(
"tenant_id %s, client_id %s, client_secret %s",
"Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]",
tenant_id,
client_id,
client_secret,
client_secret is not None,
)
if tenant_id is None:
raise ValueError(
litellm/integrations/gcs_bucket/gcs_bucket_base.py
+6 -2
View File
@@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",
@@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",
litellm/litellm_core_utils/get_llm_provider_logic.py
+2 -2
View File
@@ -202,8 +202,8 @@ def get_llm_provider( # noqa: PLR0915
)
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
raise Exception(
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
dynamic_api_key
"dynamic_api_key needs to be a string. Got type={}".format(
type(dynamic_api_key).__name__
)
)
return model, custom_llm_provider, dynamic_api_key, api_base
litellm/litellm_core_utils/llm_request_utils.py
+2 -3
View File
@@ -56,9 +56,8 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
continue
if model_info.get("mode") != "chat":
continue
_cost = model_info.get("input_cost_per_token", 0) + model_info.get(
"output_cost_per_token", 0
)
_cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get(
"output_cost_per_token") or 0.0)
model_costs.append((model, _cost))
# Sort by cost (ascending)
litellm/litellm_core_utils/prompt_templates/common_utils.py
+20 -9
View File
@@ -337,24 +337,35 @@ def _insert_assistant_continue_message(
"""
Add assistant continuation messages between consecutive user messages.
Only checks directly adjacent messages to preserve backward compatibility.
Skips tool messages and assistant messages with tool calls in the
alternation check, matching strict templates like llama.cpp.
"""
if not ensure_alternating_roles or len(messages) <= 1:
return messages
continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
# Find indexes where assistant_continue should be inserted (before that index)
insert_before_indexes: set = set()
for i in range(len(messages)):
curr = messages[i]
if _counts_for_alternation(curr) and curr["role"] == "user":
# Look backwards for the previous counted message
j = i - 1
while j >= 0:
if _counts_for_alternation(messages[j]):
if messages[j]["role"] == "user":
insert_before_indexes.add(i)
break
j -= 1
# Build the result with assistant_continue inserted at the right positions
modified_messages: List[AllMessageValues] = []
for i, message in enumerate(messages):
if (
i < len(messages) - 1
and message.get("role") == "user"
and messages[i + 1].get("role") == "user"
):
modified_messages.append(message)
if i in insert_before_indexes:
modified_messages.append(continue_message)
else:
modified_messages.append(message)
modified_messages.append(message)
return modified_messages
litellm/litellm_core_utils/streaming_handler.py
+10 -6
View File
@@ -2178,12 +2178,16 @@ class CustomStreamWrapper:
None,
)
if _deferred_cb is not None:
# Proxy has post-call guardrails — let the closure
# run guardrails on the assembled response, then
# fire logging with guardrail_information populated.
self.logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined]
asyncio.create_task(
_deferred_cb(complete_streaming_response, cache_hit)
# Proxy has post-call guardrails. Store the assembled
# response so the outer streaming consumer
# (ProxyLogging.async_post_call_streaming_iterator_hook)
# can fire the deferred callback AFTER all guardrail
# end-of-stream blocks complete. Scheduling here via
# create_task would race with unified_guardrail's
# end-of-stream block for short-stream providers.
self.logging_obj._deferred_stream_complete_args = ( # type: ignore[attr-defined]
complete_streaming_response,
cache_hit,
)
else:
asyncio.create_task(
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
+116 -138
View File
@@ -38,6 +38,102 @@ class FakeAnthropicMessagesStreamIterator:
self.chunks = self._create_streaming_chunks()
self.current_index = 0
def _create_content_block_chunks(
self, block_dict: Dict[str, Any], index: int
) -> List[bytes]:
"""Build SSE chunks for a single content block."""
chunks = []
block_type = block_dict.get("type")
if block_type == "text":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
elif block_type == "thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "thinking_delta", "thinking": thinking_text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "signature_delta", "signature": signature},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
elif block_type == "redacted_thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "input_json_delta", "partial_json": json.dumps(input_data)},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
return chunks
def _create_streaming_chunks(self) -> List[bytes]:
"""Convert the non-streaming response to streaming chunks"""
chunks = []
@@ -69,152 +165,34 @@ class FakeAnthropicMessagesStreamIterator:
# 2-4. For each content block, send start/delta/stop events
content_blocks = response_dict.get("content", [])
if content_blocks:
for index, block in enumerate(content_blocks):
# Cast block to dict for easier access
block_dict = cast(Dict[str, Any], block)
block_type = block_dict.get("type")
if block_type == "text":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send full text as one delta for simplicity)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "thinking":
# content_block_start for thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "thinking",
"thinking": "",
"signature": "",
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta for thinking text
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "thinking_delta",
"thinking": thinking_text,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_delta for signature (if present)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "signature_delta",
"signature": signature,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "redacted_thinking":
# content_block_start for redacted_thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_stop (no delta for redacted thinking)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "tool_use":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send input as JSON delta)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
for index, block in enumerate(content_blocks):
block_dict = cast(Dict[str, Any], block)
chunks.extend(self._create_content_block_chunks(block_dict, index))
# 5. message_delta event (with final usage and stop_reason)
# Include cache usage fields so clients that only read message_delta
# (like Claude Code's SDK) see the full input token breakdown.
delta_usage: Dict[str, Any] = {
"output_tokens": usage.get("output_tokens", 0) if usage else 0,
}
if usage:
if usage.get("input_tokens") is not None:
delta_usage["input_tokens"] = usage["input_tokens"]
if usage.get("cache_creation_input_tokens") is not None:
delta_usage["cache_creation_input_tokens"] = usage[
"cache_creation_input_tokens"
]
if usage.get("cache_read_input_tokens") is not None:
delta_usage["cache_read_input_tokens"] = usage[
"cache_read_input_tokens"
]
message_delta = {
"type": "message_delta",
"delta": {
"stop_reason": response_dict.get("stop_reason"),
"stop_sequence": response_dict.get("stop_sequence"),
},
"usage": {"output_tokens": usage.get("output_tokens", 0) if usage else 0},
"usage": delta_usage,
}
chunks.append(
f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode()
litellm/llms/azure/common_utils.py
+5 -9
View File
@@ -101,17 +101,15 @@ def get_azure_ad_token_from_entra_id(
_client_secret = client_secret
verbose_logger.debug(
"tenant_id %s, client_id %s, client_secret %s",
"tenant_id=%s, client_id=%s, client_secret=[set=%s]",
_tenant_id,
_client_id,
_client_secret,
_client_secret is not None,
)
if _tenant_id is None or _client_id is None or _client_secret is None:
raise ValueError("tenant_id, client_id, and client_secret must be provided")
credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
verbose_logger.debug("credential %s", credential)
token_provider = get_bearer_token_provider(credential, scope)
verbose_logger.debug("token_provider %s", token_provider)
@@ -140,10 +138,10 @@ def get_azure_ad_token_from_username_password(
from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
verbose_logger.debug(
"client_id %s, azure_username %s, azure_password %s",
"client_id=%s, azure_username=[set=%s], azure_password=[set=%s]",
client_id,
azure_username,
azure_password,
azure_username is not None,
azure_password is not None,
)
credential = UsernamePasswordCredential(
client_id=client_id,
@@ -151,8 +149,6 @@ def get_azure_ad_token_from_username_password(
password=azure_password,
)
verbose_logger.debug("credential %s", credential)
token_provider = get_bearer_token_provider(credential, scope)
verbose_logger.debug("token_provider %s", token_provider)
litellm/llms/azure_ai/cost_calculator.py
+1 -1
View File
@@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai")
router_flat_cost_per_token = model_info.get("input_cost_per_token", 0)
if router_flat_cost_per_token > 0:
if router_flat_cost_per_token and router_flat_cost_per_token > 0:
return prompt_tokens * router_flat_cost_per_token
return 0.0

Some files were not shown because too many files have changed in this diff Show More