tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-14 08:26:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into litellm_perf_convert_model_response_frozensets

Browse Source 
Resolve conflict: keep main's provider_specific_fields passthrough
preservation while using frozenset set-difference optimization.
This commit is contained in:
Ryan Crabbe
2026-02-21 14:52:43 -08:00
parent c039e505ba 8c7f667df2
commit bb24ebebd9
1476 changed files with 135132 additions and 33394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+90 -16
View File
@@ -21,9 +21,7 @@ commands:
- run:
name: "Install local version of litellm-enterprise"
command: |
cd enterprise
python -m pip install -e .
cd ..
pip install --force-reinstall --no-deps -e enterprise/
setup_litellm_test_deps:
steps:
- checkout
@@ -1458,6 +1456,7 @@ jobs:
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.36.0"
pip install "semantic_router==0.1.10"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1656,7 +1655,7 @@ jobs:
- search_coverage.xml
- search_coverage
# Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
litellm_mapped_tests_proxy:
litellm_mapped_tests_proxy_part1:
docker:
- image: cimg/python:3.11
auth:
@@ -1667,23 +1666,53 @@ jobs:
steps:
- setup_litellm_test_deps
- run:
name: Run proxy tests
name: Run proxy tests part 1 (high-volume directories)
command: |
prisma generate
python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_tests_coverage.xml
mv .coverage litellm_proxy_tests_coverage
mv coverage.xml litellm_proxy_tests_part1_coverage.xml
mv .coverage litellm_proxy_tests_part1_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_tests_coverage.xml
- litellm_proxy_tests_coverage
- litellm_proxy_tests_part1_coverage.xml
- litellm_proxy_tests_part1_coverage
litellm_mapped_tests_proxy_part2:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run proxy tests part 2 (all other tests)
command: |
prisma generate
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_tests_part2_coverage.xml
mv .coverage litellm_proxy_tests_part2_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_tests_part2_coverage.xml
- litellm_proxy_tests_part2_coverage
litellm_mapped_tests_llms:
docker:
- image: cimg/python:3.11
@@ -1724,7 +1753,7 @@ jobs:
- run:
name: Run core tests
command: |
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1765,6 +1794,33 @@ jobs:
paths:
- litellm_core_utils_tests_coverage.xml
- litellm_core_utils_tests_coverage
litellm_mapped_tests_mcps:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run MCP client tests
command: |
python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mcps_tests_coverage.xml
mv .coverage litellm_mcps_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mcps_tests_coverage.xml
- litellm_mcps_tests_coverage
litellm_mapped_tests_integrations:
docker:
- image: cimg/python:3.11
@@ -3597,9 +3653,11 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
@@ -3652,7 +3710,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@@ -4392,7 +4450,13 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_proxy:
- litellm_mapped_tests_proxy_part1:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_proxy_part2:
filters:
branches:
only:
@@ -4410,6 +4474,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_mcps:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_integrations:
filters:
branches:
@@ -4469,9 +4539,11 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests_proxy
- litellm_mapped_tests_proxy_part1
- litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
@@ -4548,9 +4620,11 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests_proxy
- litellm_mapped_tests_proxy_part1
- litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
.claude/settings.json
+36
View File
@@ -0,0 +1,36 @@
{
"permissions": {
"allow": [
"Bash(git show:*)",
"Bash(git worktree add:*)",
"Read(//Users/krrishdholakia/Documents/litellm/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
"Bash(python:*)",
"Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
"Bash(poetry run pytest:*)",
"Bash(git add:*)",
"Bash(git commit:*)",
"Bash(poetry run python:*)",
"Bash(poetry run pip:*)",
"Bash(git reset:*)",
"Bash(git cherry-pick:*)",
"Bash(git checkout:*)",
"Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
"Read(//Users/krrishdholakia/Documents/**)",
"Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
"Bash(ls:*)"
],
"additionalDirectories": [
"/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
]
}
}
.github/actions/helm-oci-chart-releaser/action.yml
+7 -12
View File
@@ -40,38 +40,33 @@ outputs:
runs:
using: composite
steps:
- name: Helm | Setup
uses: azure/setup-helm@v4
with:
version: v3.20.0
- name: Helm | Login
shell: bash
run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Package
shell: bash
run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Push
shell: bash
run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Logout
shell: bash
run: helm registry logout ${{ inputs.registry }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Output
id: output
shell: bash
run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
.github/workflows/check_duplicate_issues.yml
+37 -18
View File
@@ -2,28 +2,47 @@ name: Check Duplicate Issues
on:
issues:
types: [opened, edited]
types: [opened]
jobs:
check-duplicate:
check-duplicates:
if: github.event.action == 'opened'
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
issues: write
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
- name: Install Claude Code
run: npm install -g @anthropic-ai/claude-code
- name: Check duplicates
env:
ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }}
ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
PROMPT: |
A new issue has been created in the ${{ github.repository }} repository.
Issue number: ${{ github.event.issue.number }}
Lookup this issue with gh issue view ${{ github.event.issue.number }} --repo ${{ github.repository }}.
Search through existing issues (excluding #${{ github.event.issue.number }}) to find potential duplicates.
Use gh issue list --repo ${{ github.repository }} with relevant search terms from the new issue's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed issues.
Consider:
1. Similar titles or descriptions
2. Same error messages or symptoms
3. Related functionality or components
4. Similar feature requests
If you find potential duplicates, post a SINGLE comment on issue #${{ github.event.issue.number }} using gh issue comment ${{ github.event.issue.number }} --repo ${{ github.repository }} with this format:
_This comment was generated by an LLM and may be inaccurate._
This issue might be a duplicate of existing issues. Please check:
- #[issue_number]: [brief description of similarity]
If you find NO duplicates, do NOT post any comment. Stay silent.
run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh issue *)"
.github/workflows/check_duplicate_prs.yml
+52
View File
@@ -0,0 +1,52 @@
name: Check Duplicate PRs
on:
pull_request_target:
types: [opened]
jobs:
check-duplicates:
if: |
github.event.pull_request.user.login != 'ishaan-jaff' &&
github.event.pull_request.user.login != 'krrishdholakia' &&
github.event.pull_request.user.login != 'actions-user' &&
!endsWith(github.event.pull_request.user.login, '[bot]')
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: Install Claude Code
run: npm install -g @anthropic-ai/claude-code
- name: Check duplicates
env:
ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }}
ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROMPT: |
A new PR has been opened in the ${{ github.repository }} repository.
PR number: ${{ github.event.pull_request.number }}
Lookup this PR with gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }}.
Search through existing open PRs (excluding #${{ github.event.pull_request.number }}) to find potential duplicates.
Use gh pr list --repo ${{ github.repository }} with relevant search terms from the new PR's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed PRs.
Consider:
1. Similar titles or descriptions
2. Same bug fix or feature being implemented
3. Related functionality or components
4. Overlapping code changes (same files or areas)
If you find potential duplicates, post a SINGLE comment on PR #${{ github.event.pull_request.number }} using gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} with this format:
_This comment was generated by an LLM and may be inaccurate._
This PR might be a duplicate of existing PRs. Please check:
- #[pr_number]: [brief description of similarity]
If you find NO duplicates, do NOT post any comment. Stay silent.
run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh pr *)"
.github/workflows/regenerate-poetry-lock.yml
+80
View File
@@ -0,0 +1,80 @@
name: Regenerate poetry.lock
# Runs whenever pyproject.toml is merged into main (the most common cause of
# the "pyproject.toml changed significantly since poetry.lock was last generated"
# CI failure). Can also be triggered manually.
on:
push:
branches:
- main
paths:
- pyproject.toml
workflow_dispatch:
permissions:
contents: write # needed to push the auto/regenerate-poetry-lock-* branch
pull-requests: write # needed to open the PR and enable auto-merge
jobs:
regenerate-lock:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Regenerate poetry.lock
run: poetry lock
- name: Check whether poetry.lock actually changed
id: diff
run: |
if git diff --quiet poetry.lock; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Open PR with the refreshed lock file
if: steps.diff.outputs.changed == 'true'
id: open-pr
run: |
BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git add poetry.lock
git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
git push -f origin "$BRANCH"
cat > /tmp/pr-body.md << 'BODY'
Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
Fixes the recurring CI failure:
```
pyproject.toml changed significantly since poetry.lock was last generated.
Run `poetry lock` to fix the lock file.
```
BODY
PR_URL=$(gh pr create \
--title "chore: regenerate poetry.lock to match pyproject.toml" \
--body-file /tmp/pr-body.md \
--head "$BRANCH" \
--base main)
echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
if: steps.diff.outputs.changed == 'true'
run: |
gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}
.github/workflows/test-litellm-matrix.yml
+89 -32
View File
@@ -12,44 +12,107 @@ concurrency:
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 15
timeout-minutes: 20 # Increased from 15 to 20
strategy:
fail-fast: false
matrix:
test-group:
# tests/test_litellm split by subdirectory (~560 files total)
- name: "llms"
path: "tests/test_litellm/llms"
workers: 4
# Vertex AI tests separated for better isolation (prevent auth/env pollution)
- name: "llms-vertex"
path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
- name: "llms-other"
path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
# tests/test_litellm/proxy split by subdirectory (~180 files total)
- name: "proxy-guardrails"
path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
workers: 4
workers: 2
reruns: 2
- name: "proxy-core"
path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
workers: 4
workers: 2
reruns: 2
- name: "proxy-misc"
path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
workers: 4
workers: 2
reruns: 2
- name: "integrations"
path: "tests/test_litellm/integrations"
workers: 4
workers: 2
reruns: 3 # Integration tests tend to be flakier
- name: "core-utils"
path: "tests/test_litellm/litellm_core_utils"
workers: 2
- name: "other"
path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
workers: 4
reruns: 1
- name: "other-1"
# responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
- name: "other-2"
# enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
workers: 2
reruns: 2
- name: "other-3"
# remaining dirs ≈ 8.0k lines
path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
workers: 2
reruns: 2
- name: "root"
path: "tests/test_litellm/test_*.py"
workers: 4
workers: 2
reruns: 2
# tests/proxy_unit_tests split alphabetically (~48 files total)
- name: "proxy-unit-a"
path: "tests/proxy_unit_tests/test_[a-o]*.py"
- name: "proxy-unit-a1"
# test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
path: "tests/proxy_unit_tests/test_[a-j]*.py"
workers: 2
- name: "proxy-unit-b"
path: "tests/proxy_unit_tests/test_[p-z]*.py"
reruns: 1
- name: "proxy-unit-a2"
# test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
path: "tests/proxy_unit_tests/test_[k-o]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b1"
# lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b2"
# proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
path: "tests/proxy_unit_tests/test_proxy_server.py"
workers: 2
reruns: 1
- name: "proxy-unit-b3"
# proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
workers: 2
reruns: 1
- name: "proxy-unit-b4"
# proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
path: "tests/proxy_unit_tests/test_proxy_utils.py"
workers: 2
reruns: 1
- name: "proxy-unit-b5"
# proxy_token_counter (1279) - runs independently from utils
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
workers: 2
reruns: 1
- name: "proxy-unit-b6"
# test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
path: "tests/proxy_unit_tests/test_[r-t]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b7"
# test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
path: "tests/proxy_unit_tests/test_[u-z]*.py"
workers: 2
reruns: 1
name: test (${{ matrix.test-group.name }})
@@ -79,12 +142,17 @@ jobs:
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \
# pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
poetry run pip install google-genai==1.22.0 \
google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
- name: Setup litellm-enterprise
run: |
cd enterprise && poetry run pip install -e . && cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
run: |
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
run: |
@@ -92,18 +160,7 @@ jobs:
--tb=short -vv \
--maxfail=10 \
-n ${{ matrix.test-group.workers }} \
--reruns ${{ matrix.test-group.reruns }} \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
# Aggregate job to require all matrix jobs pass
test-complete:
needs: test
runs-on: ubuntu-latest
if: always()
steps:
- name: Check test results
run: |
if [ "${{ needs.test.result }}" != "success" ]; then
echo "Some test groups failed"
exit 1
fi
echo "All test groups passed!"
.github/workflows/test-litellm-ui-build.yml
+32
View File
@@ -0,0 +1,32 @@
name: UI Build Check
permissions:
contents: read
on:
pull_request:
branches: [main]
jobs:
build-ui:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
run: npm install
- name: Build
run: npm run build
.github/workflows/test-litellm.yml
+1 -3
View File
@@ -42,9 +42,7 @@ jobs:
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
poetry run pip install -e .
cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50
.github/workflows/test-mcp.yml
+1 -3
View File
@@ -40,9 +40,7 @@ jobs:
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run MCP tests
run: |
.github/workflows/test_server_root_path.yml
+96
View File
@@ -0,0 +1,96 @@
name: Test Proxy SERVER_ROOT_PATH Routing
permissions:
contents: read
on:
pull_request:
branches: [main]
jobs:
test-server-root-path:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
matrix:
root_path: ["/api/v1", "/llmproxy"]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile.non_root
tags: litellm-test:${{ github.sha }}
load: true
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Start LiteLLM container with SERVER_ROOT_PATH
run: |
docker run -d \
--name litellm-test \
-p 4000:4000 \
-e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
-e LITELLM_MASTER_KEY="sk-1234" \
litellm-test:${{ github.sha }} \
--detailed_debug
- name: Wait for container to be healthy
run: |
echo "Waiting for LiteLLM to start..."
max_attempts=30
attempt=0
while [ $attempt -lt $max_attempts ]; do
if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
echo "LiteLLM started successfully"
break
fi
attempt=$((attempt + 1))
echo "Attempt $attempt/$max_attempts - waiting for server to start..."
sleep 2
done
if [ $attempt -eq $max_attempts ]; then
echo "Server failed to start within timeout"
docker logs litellm-test
exit 1
fi
sleep 5
- name: Show container logs
if: always()
run: docker logs litellm-test
- name: Test UI endpoint with root path
run: |
ROOT_PATH="${{ matrix.root_path }}"
echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
for i in 1 2 3; do
content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
echo "UI page contains valid HTML content"
exit 0
fi
echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
sleep 5
done
echo "UI page does not contain expected HTML content"
echo "Response: $content"
docker logs litellm-test
exit 1
- name: Cleanup
if: always()
run: |
docker stop litellm-test || true
docker rm litellm-test || true
.gitignore
+1
View File
@@ -2,6 +2,7 @@
.venv
.venv_policy_test
.env
.claude
.newenv
newenv/*
litellm/proxy/myenv/*
ci_cd/security_scans.sh
+2
View File
@@ -158,6 +158,8 @@ run_grype_scans() {
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
)
# Build JSON array of allowlisted CVE IDs for jq
cookbook/mock_prompt_management_server/README.md
+293
View File
@@ -0,0 +1,293 @@
# Mock Prompt Management Server
A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
## Quick Start
### 1. Install Dependencies
```bash
pip install fastapi uvicorn pydantic
```
### 2. Start the Server
```bash
python mock_prompt_management_server.py
```
The server will start on `http://localhost:8080`
### 3. Test the Endpoint
```bash
# Get a prompt
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
# Get a prompt with authentication
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
-H "Authorization: Bearer test-token-12345"
# List all prompts
curl "http://localhost:8080/prompts"
# Get prompt variables
curl "http://localhost:8080/prompts/hello-world-prompt/variables"
```
## Using with LiteLLM
### Configuration
Create a `config.yaml` file:
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
prompts:
- prompt_id: "hello-world-prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
api_key: test-token-12345
```
### Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### Make a Request
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"prompt_id": "hello-world-prompt",
"prompt_variables": {
"domain": "data science",
"task": "analyzing customer behavior"
},
"messages": [
{"role": "user", "content": "Please help me get started"}
]
}'
```
## Available Prompts
The server includes several example prompts:
| Prompt ID | Description | Variables |
|-----------|-------------|-----------|
| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
## Authentication
The server supports optional Bearer token authentication. Valid tokens for testing:
- `test-token-12345`
- `dev-token-67890`
- `prod-token-abcdef`
If no `Authorization` header is provided, requests are allowed (for testing purposes).
## API Endpoints
### LiteLLM Spec Endpoints
#### `GET /beta/litellm_prompt_management`
Get a prompt by ID (required by LiteLLM).
**Query Parameters:**
- `prompt_id` (required): The prompt ID
- `project_name` (optional): Project filter
- `slug` (optional): Slug filter
- `version` (optional): Version filter
**Response:**
```json
{
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
### Convenience Endpoints (Not in LiteLLM Spec)
#### `GET /health`
Health check endpoint.
#### `GET /prompts`
List all available prompts.
#### `GET /prompts/{prompt_id}/variables`
Get all variables used in a prompt template.
#### `POST /prompts`
Create a new prompt (in-memory only, for testing).
## Example: Full Integration Test
### 1. Start the Mock Server
```bash
python mock_prompt_management_server.py
```
### 2. Test with Python
```python
from litellm import completion
# The completion will:
# 1. Fetch the prompt from your API
# 2. Replace {domain} with "machine learning"
# 3. Replace {task} with "building a recommendation system"
# 4. Merge with your messages
# 5. Use the model and params from the prompt
response = completion(
model="gpt-4",
prompt_id="hello-world-prompt",
prompt_variables={
"domain": "machine learning",
"task": "building a recommendation system"
},
messages=[
{"role": "user", "content": "I have user behavior data from the past year."}
],
# Configure the generic prompt manager
generic_prompt_config={
"api_base": "http://localhost:8080",
"api_key": "test-token-12345",
}
)
print(response.choices[0].message.content)
```
## Customization
### Adding New Prompts
Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
```python
PROMPTS_DB = {
"my-custom-prompt": {
"prompt_id": "my-custom-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a {role}."
},
{
"role": "user",
"content": "{user_input}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.8,
"max_tokens": 1000
}
}
}
```
### Using a Database
Replace the `PROMPTS_DB` dictionary with database queries:
```python
@app.get("/beta/litellm_prompt_management")
async def get_prompt(prompt_id: str):
# Fetch from database
prompt = await db.prompts.find_one({"prompt_id": prompt_id})
if not prompt:
raise HTTPException(status_code=404, detail="Prompt not found")
return PromptResponse(**prompt)
```
### Adding Access Control
Use the custom query parameters for access control:
```python
@app.get("/beta/litellm_prompt_management")
async def get_prompt(
prompt_id: str,
project_name: Optional[str] = None,
user_id: Optional[str] = None,
authorization: Optional[str] = Header(None)
):
token = verify_api_key(authorization)
# Check if user has access to this project
if not has_project_access(token, project_name):
raise HTTPException(status_code=403, detail="Access denied")
# Fetch and return prompt
...
```
## Production Considerations
Before deploying to production:
1. **Use a real database** instead of in-memory storage
2. **Implement proper authentication** with JWT tokens or API keys
3. **Add rate limiting** to prevent abuse
4. **Use HTTPS** for encrypted communication
5. **Add logging and monitoring** for observability
6. **Implement caching** for frequently accessed prompts
7. **Add versioning** for prompt management
8. **Implement access control** based on teams/users
9. **Add input validation** for all parameters
10. **Use environment variables** for configuration
## Related Documentation
- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
## Questions?
This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
cookbook/mock_prompt_management_server/mock_prompt_management_server.py
+390
View File
@@ -0,0 +1,390 @@
#!/usr/bin/env python3
"""
Mock Prompt Management API Server
This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
for testing and demonstration purposes.
Usage:
python mock_prompt_management_server.py
The server will start on http://localhost:8080
Test the endpoint:
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
"""
import os
import json
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, Header, Query, status
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
# ============================================================================
# Response Models
# ============================================================================
class MessageContent(BaseModel):
"""A single message in the prompt template"""
role: str = Field(..., description="Message role (system, user, assistant)")
content: str = Field(
..., description="Message content with optional {variable} placeholders"
)
class PromptResponse(BaseModel):
"""Response format for the prompt management API"""
prompt_id: str = Field(..., description="The ID of the prompt")
prompt_template: List[MessageContent] = Field(
..., description="Array of messages in OpenAI format"
)
prompt_template_model: Optional[str] = Field(
None, description="Optional model to use for this prompt"
)
prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
None, description="Optional parameters like temperature, max_tokens, etc."
)
# ============================================================================
# Mock Prompt Database
# ============================================================================
PROMPTS_DB = {
"hello-world-prompt": {
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}.",
},
{"role": "user", "content": "Help me with: {task}"},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
},
"code-review-prompt": {
"prompt_id": "code-review-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
},
{
"role": "user",
"content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
},
],
"prompt_template_model": "gpt-4-turbo",
"prompt_template_optional_params": {
"temperature": 0.3,
"max_tokens": 1500,
},
},
"customer-support-prompt": {
"prompt_id": "customer-support-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
},
{
"role": "user",
"content": "Customer inquiry: {customer_message}",
},
],
"prompt_template_model": "gpt-3.5-turbo",
"prompt_template_optional_params": {
"temperature": 0.8,
"max_tokens": 800,
"top_p": 0.9,
},
},
"data-analysis-prompt": {
"prompt_id": "data-analysis-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a data scientist expert in {analysis_type} analysis.",
},
{
"role": "user",
"content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.5,
"max_tokens": 2000,
},
},
"creative-writing-prompt": {
"prompt_id": "creative-writing-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a creative writer specializing in {genre} fiction.",
},
{
"role": "user",
"content": "Write a {length} story about: {topic}",
},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.9,
"max_tokens": 3000,
"top_p": 0.95,
},
},
}
# Valid API tokens for authentication (in production, use a secure token store)
VALID_API_TOKENS = {
"test-token-12345",
"dev-token-67890",
"prod-token-abcdef",
}
# ============================================================================
# FastAPI App
# ============================================================================
app = FastAPI(
title="Mock Prompt Management API",
description="A mock server implementing the LiteLLM Generic Prompt Management API",
version="1.0.0",
)
def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
"""
Verify the API key from the Authorization header.
Args:
authorization: Authorization header (Bearer token)
Returns:
True if valid, raises HTTPException if invalid
"""
if authorization is None:
# Allow requests without authentication for testing
return True
# Extract token from "Bearer <token>"
if not authorization.startswith("Bearer "):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authorization header format. Expected 'Bearer <token>'",
)
token = authorization.replace("Bearer ", "").strip()
if token not in VALID_API_TOKENS:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid API key",
)
return True
@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
async def get_prompt(
prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
project_name: Optional[str] = Query(
None, description="Optional project name filter"
),
slug: Optional[str] = Query(None, description="Optional slug filter"),
version: Optional[str] = Query(None, description="Optional version filter"),
authorization: Optional[str] = Header(None),
) -> PromptResponse:
"""
Get a prompt by ID with optional filtering.
This endpoint implements the LiteLLM Generic Prompt Management API specification.
Args:
prompt_id: The ID of the prompt to fetch
project_name: Optional project name for filtering
slug: Optional slug for filtering
version: Optional version for filtering
authorization: Optional Bearer token for authentication
Returns:
PromptResponse with the prompt template and configuration
Raises:
HTTPException: 401 if authentication fails, 404 if prompt not found
"""
# Verify authentication
verify_api_key(authorization)
# Log the request parameters (useful for debugging)
print(f"Fetching prompt: {prompt_id}")
if project_name:
print(f" Project: {project_name}")
if slug:
print(f" Slug: {slug}")
if version:
print(f" Version: {version}")
# Check if prompt exists
if prompt_id not in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
)
# Get the prompt from the database
prompt_data = PROMPTS_DB[prompt_id]
# Optional: Apply filtering based on project_name, slug, or version
# In a real implementation, you might use these to filter prompts by access control
# or to fetch specific versions from your database
return PromptResponse(**prompt_data)
@app.get("/health")
async def health_check():
"""Health check endpoint"""
return {
"status": "healthy",
"service": "mock-prompt-management-api",
"version": "1.0.0",
}
@app.get("/prompts")
async def list_prompts(authorization: Optional[str] = Header(None)):
"""
List all available prompts.
This is a convenience endpoint (not part of the LiteLLM spec) for
discovering available prompts.
"""
# Verify authentication
verify_api_key(authorization)
prompts_list = [
{
"prompt_id": pid,
"model": p.get("prompt_template_model"),
"has_variables": any(
"{" in msg.get("content", "") for msg in p.get("prompt_template", [])
),
}
for pid, p in PROMPTS_DB.items()
]
return {"prompts": prompts_list, "total": len(prompts_list)}
@app.get("/prompts/{prompt_id}/variables")
async def get_prompt_variables(
prompt_id: str, authorization: Optional[str] = Header(None)
):
"""
Get all variables in a prompt template.
This is a convenience endpoint (not part of the LiteLLM spec) for
discovering what variables a prompt expects.
"""
# Verify authentication
verify_api_key(authorization)
if prompt_id not in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Prompt '{prompt_id}' not found",
)
prompt_data = PROMPTS_DB[prompt_id]
variables = set()
# Extract variables from the prompt template
import re
for message in prompt_data["prompt_template"]:
content = message.get("content", "")
# Find all {variable} patterns
found_vars = re.findall(r"\{(\w+)\}", content)
variables.update(found_vars)
return {
"prompt_id": prompt_id,
"variables": sorted(list(variables)),
"example_usage": {
"prompt_id": prompt_id,
"prompt_variables": {var: f"<{var}_value>" for var in variables},
},
}
@app.post("/prompts")
async def create_prompt(
prompt: PromptResponse, authorization: Optional[str] = Header(None)
):
"""
Create a new prompt (convenience endpoint for testing).
This is NOT part of the LiteLLM spec - it's just for testing purposes.
"""
# Verify authentication
verify_api_key(authorization)
if prompt.prompt_id in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Prompt '{prompt.prompt_id}' already exists",
)
PROMPTS_DB[prompt.prompt_id] = prompt.dict()
return {
"status": "created",
"prompt_id": prompt.prompt_id,
"message": "Prompt created successfully (in-memory only)",
}
# ============================================================================
# Main
# ============================================================================
if __name__ == "__main__":
import uvicorn
print("=" * 70)
print("Mock Prompt Management API Server")
print("=" * 70)
print(f"\nStarting server on http://localhost:8080")
print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
for prompt_id in PROMPTS_DB.keys():
print(f" - {prompt_id}")
print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
print(" - test-token-12345")
print(" - dev-token-67890")
print(" - prod-token-abcdef")
print("\nEndpoints:")
print(" GET /beta/litellm_prompt_management?prompt_id=<id> (LiteLLM spec)")
print(" GET /health (health check)")
print(" GET /prompts (list all prompts)")
print(
" GET /prompts/{id}/variables (get prompt variables)"
)
print(" POST /prompts (create prompt)")
print("\nExample usage:")
print(
' curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
)
print("\nPress CTRL+C to stop the server")
print("=" * 70)
uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")
deploy/charts/litellm-helm/Chart.yaml
+4
View File
@@ -26,6 +26,10 @@ version: 1.1.0
# It is recommended to use it with quotes.
appVersion: v1.80.12
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
org.opencontainers.image.url: "https://docs.litellm.ai/"
dependencies:
- name: "postgresql"
version: ">=13.3.0"
docs/my-website/blog/claude_code_beta_headers/index.md
+177
View File
@@ -0,0 +1,177 @@
---
slug: claude-code-beta-headers-incident
title: "Incident Report: Invalid beta headers with Claude Code"
date: 2026-02-16T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
tags: [incident-report, anthropic, stability]
hide_table_of_contents: false
---
**Date:** February 13, 2026
**Duration:** ~3 hours
**Severity:** High
**Status:** Resolved
> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
## Summary
Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
- **LLM calls to Anthropic:** No impact.
- **LLM calls to Bedrock/Azure/Vertex:** Failed with `invalid beta flag` errors when unsupported headers were present.
- **Cost tracking and routing:** No impact.
{/* truncate */}
---
## Background
Anthropic uses beta headers to enable experimental features in Claude. When Claude Code makes API requests, it includes headers like `anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20`. However, not all providers support all Anthropic beta features.
Before this incident, LiteLLM forwarded all beta headers to all providers without validation:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM (old behavior)
participant Provider as Provider (Bedrock/Azure/Vertex)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Provider: Forward ALL headers (no validation)
Note over LP,Provider: anthropic-beta: header1,header2,header3
Provider-->>LP: ❌ Error: invalid beta flag
LP-->>CC: Request fails
```
Requests succeeded for Anthropic (native support) but failed for other providers when Claude Code sent headers those providers didn't support.
---
## Root cause
LiteLLM lacked provider-specific beta header validation. When Claude Code introduced new beta features or sent headers that specific providers didn't support, those headers were blindly forwarded, causing provider API errors.
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Create `anthropic_beta_headers_config.json` with provider-specific mappings | ✅ Done | [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) |
| 2 | Implement strict validation: headers must be explicitly mapped to be forwarded | ✅ Done | [`litellm_logging.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/litellm_logging.py) |
| 3 | Add `/reload/anthropic_beta_headers` endpoint for dynamic config updates | ✅ Done | Proxy management endpoints |
| 4 | Add `/schedule/anthropic_beta_headers_reload` for automatic periodic updates | ✅ Done | Proxy management endpoints |
| 5 | Support `LITELLM_ANTHROPIC_BETA_HEADERS_URL` for custom config sources | ✅ Done | Environment configuration |
| 6 | Support `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` for air-gapped deployments | ✅ Done | Environment configuration |
Now LiteLLM validates and transforms headers per-provider:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM (new behavior)
participant Config as Beta Headers Config
participant Provider as Provider (Bedrock/Azure/Vertex)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Config: Load header mapping for provider
Config-->>LP: Returns mapping (header→value or null)
Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
LP->>Provider: Request with filtered & mapped headers
Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
Provider-->>LP: ✅ Success response
LP-->>CC: Response
```
---
## Dynamic configuration updates
A key improvement is zero-downtime configuration updates. When Anthropic releases new beta features, users can update their configuration without restarting:
```bash
# Manually trigger reload (no restart needed)
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
# Or schedule automatic reloads every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
This prevents future incidents where Claude Code introduces new headers before LiteLLM configuration is updated.
---
## Configuration format
The `anthropic_beta_headers_config.json` file maps input headers to provider-specific output headers:
```json
{
"description": "Mapping of Anthropic beta headers for each provider.",
"anthropic": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"computer-use-2025-01-24": "computer-use-2025-01-24"
},
"bedrock_converse": {
"advanced-tool-use-2025-11-20": null,
"computer-use-2025-01-24": "computer-use-2025-01-24"
},
"azure_ai": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"computer-use-2025-01-24": "computer-use-2025-01-24"
}
}
```
**Validation rules:**
1. Headers must exist in the mapping for the target provider
2. Headers with `null` values are filtered out (unsupported)
3. Header names can be transformed per-provider (e.g., Bedrock uses different names for some features)
---
## Resolution steps for users
For users still experiencing issues, update to the latest LiteLLM version if < v1.81.11-nightly:
```bash
pip install --upgrade litellm
```
Or manually reload the configuration without restarting:
```bash
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
---
## Related documentation
- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file
docs/my-website/blog/claude_opus_4_6/index.md
+20 -1
View File
@@ -185,7 +185,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: bedrock/anthropic.claude-opus-4-6-v1:0
model: bedrock/anthropic.claude-opus-4-6-v1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@@ -389,6 +389,10 @@ Compaction blocks are also supported in streaming mode. You'll receive:
### Adaptive Thinking
:::note
When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below).
:::
<Tabs>
<TabItem value="completions" label="/chat/completions">
@@ -434,6 +438,21 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \
}'
```
</TabItem>
<TabItem value="native" label="Native thinking param">
Use the `thinking` parameter directly for adaptive thinking via the SDK:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}],
thinking={"type": "adaptive"},
)
```
</TabItem>
</Tabs>
docs/my-website/blog/claude_sonnet_4_6/index.md
+283
View File
@@ -0,0 +1,283 @@
---
slug: claude_sonnet_4_6
title: "Day 0 Support: Claude Sonnet 4.6"
date: 2026-02-17T10:00:00
authors:
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, sonnet 4.6]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
```
## Usage - Anthropic
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: anthropic/claude-sonnet-4-6
api_key: os.environ/ANTHROPIC_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="anthropic/claude-sonnet-4-6",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Azure
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: azure_ai/claude-sonnet-4-6
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE # https://<resource>.services.ai.azure.com
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="azure_ai/claude-sonnet-4-6",
api_key="your-azure-api-key",
api_base="https://<resource>.services.ai.azure.com",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Vertex AI
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: vertex_ai/claude-sonnet-4-6
vertex_project: os.environ/VERTEX_PROJECT
vertex_location: us-east5
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e VERTEX_PROJECT=$VERTEX_PROJECT \
-e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/credentials.json:/app/credentials.json \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/claude-sonnet-4-6",
vertex_project="your-project-id",
vertex_location="us-east5",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Bedrock
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: bedrock/anthropic.claude-sonnet-4-6-v1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="bedrock/anthropic.claude-sonnet-4-6-v1",
aws_access_key_id="your-access-key",
aws_secret_access_key="your-secret-key",
aws_region_name="us-east-1",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
docs/my-website/blog/gemin_3.1/index.md
+150
View File
@@ -0,0 +1,150 @@
---
slug: gemini_3_1_pro
title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
date: 2026-02-19T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini 3.1 Pro Day 0 Support
LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==v1.81.9-stable.gemini.3.1-pro
```
</TabItem>
</Tabs>
## What's New
### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
Gemini 3.1 Pro introduces support for **medium** thinking level
LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
---
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
- Conversion of provider specific thinking related param to thinkingLevel
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage with MEDIUM thinking (NEW)**
```python
from litellm import completion
# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
response = completion(
model="gemini/gemini-3.1-pro-preview",
messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
reasoning_effort="medium", # NEW: MEDIUM thinking level
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gemini-3.1-pro-preview
litellm_params:
model: gemini/gemini-3.1-pro-preview
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-gemini-3.1-pro-preview
litellm_params:
model: vertex_ai/gemini-3.1-pro-preview
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Call with MEDIUM thinking**
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3.1-pro-preview",
"messages": [{"role": "user", "content": "Complex reasoning task"}],
"reasoning_effort": "medium"
}'
```
</TabItem>
</Tabs>
---
## `reasoning_effort` Mapping for Gemini 3+
| reasoning_effort | thinking_level |
|------------------|----------------|
| `minimal` | `minimal` |
| `low` | `low` |
| `medium` | `medium` |
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |
docs/my-website/blog/vllm_embeddings_incident/index.md
+117
View File
@@ -0,0 +1,117 @@
---
slug: vllm-embeddings-incident
title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
date: 2026-02-18T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, embeddings, vllm]
hide_table_of_contents: false
---
**Date:** Feb 16, 2026
**Duration:** ~3 hours
**Severity:** High (for vLLM embedding users)
**Status:** Resolved
## Summary
A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
- **vLLM embedding calls:** Complete failure - all requests rejected
- **Other providers:** No impact - OpenAI and other providers functioned normally
- **Other vLLM functionality:** No impact - only embeddings were affected
{/* truncate */}
---
## Background
The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
```mermaid
flowchart TD
A["1. User calls litellm.embedding()
litellm/main.py"] --> B["2. Transform request for provider
litellm/llms/openai_like/embedding/handler.py"]
B --> C["3. Send request to vLLM endpoint"]
C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
C -->|"encoding_format='float' or 'base64'"| D
C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
'unknown variant, expected float or base64'"]
style D fill:#d4edda,stroke:#28a745
style E fill:#f8d7da,stroke:#dc3545
style B fill:#fff3cd,stroke:#ffc107
```
---
## Root cause
A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
```python
# Added in dbcae4a
if encoding_format is not None:
optional_params["encoding_format"] = encoding_format
else:
# Omitting causes openai sdk to add default value of "float"
optional_params["encoding_format"] = None
```
This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
---
## The Fix
Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
**In `litellm/llms/openai_like/embedding/handler.py`:**
```python
# Before (broken)
data = {"model": model, "input": input, **optional_params}
# After (fixed)
filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
data = {"model": model, "input": input, **filtered_optional_params}
```
This ensures:
- Valid values (`"float"`, `"base64"`) are preserved and sent
- `None` and empty string values are filtered out (parameter omitted entirely)
- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
---
docs/my-website/docs/adding_provider/generic_guardrail_api.md
+1
View File
@@ -237,6 +237,7 @@ litellm_settings:
mode: pre_call # or post_call, during_call
api_base: https://your-guardrail-api.com
api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional
unreachable_fallback: fail_closed # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
additional_provider_specific_params:
# your custom parameters
threshold: 0.8
docs/my-website/docs/adding_provider/generic_prompt_management_api.md
+576
View File
@@ -0,0 +1,576 @@
# [BETA] Generic Prompt Management API - Integrate Without a PR
## The Problem
As a prompt management provider, integrating with LiteLLM traditionally requires:
- Making a PR to the LiteLLM repository
- Waiting for review and merge
- Maintaining provider-specific code in LiteLLM's codebase
- Updating the integration for changes to your API
## The Solution
The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
### Key Benefits
1. **No PR Needed** - Deploy and integrate immediately
3. **Simple Contract** - One GET endpoint, standard JSON response
4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
5. **Custom Parameters** - Pass provider-specific query params via config
6. **Full Control** - You own and maintain your prompt management API
7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
## Get Started in 3 Steps
### Step 1: Configure LiteLLM
Add to your `config.yaml`:
```yaml
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
api_key: os.environ/YOUR_API_KEY
```
### Step 2: Implement Your API Endpoint
```python
from fastapi import FastAPI
from pydantic import BaseModel
app = FastAPI()
@app.get("/beta/litellm_prompt_management")
async def get_prompt(prompt_id: str):
return {
"prompt_id": prompt_id,
"prompt_template": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Help me with {task}"}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {"temperature": 0.7}
}
```
### Step 3: Use in Your App
```python
from litellm import completion
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
prompt_variables={"task": "data analysis"},
messages=[{"role": "user", "content": "I have sales data"}]
)
```
That's it! LiteLLM fetches your prompt, applies variables, and makes the request
## API Contract
### Endpoint
Implement `GET /beta/litellm_prompt_management`
### Request Format
Your endpoint will receive a GET request with query parameters:
```
GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
```
**Query Parameters:**
- `prompt_id` (required): The ID of the prompt to fetch
- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
**Example:**
```
GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
```
### Response Format
```json
{
"prompt_id": "hello-world-prompt-2bac",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500,
"top_p": 0.9
}
}
```
**Response Fields:**
- `prompt_id` (string, required): The ID of the prompt
- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
## LiteLLM Configuration
Add to `config.yaml`:
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
provider_specific_query_params:
project_name: litellm
slug: hello-world-prompt-2bac
api_base: http://localhost:8080
api_key: os.environ/YOUR_PROMPT_API_KEY # optional
ignore_prompt_manager_model: true # optional, keep client's model
ignore_prompt_manager_optional_params: true # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
```
### Configuration Parameters
- `prompt_integration`: Must be `"generic_prompt_management"`
- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
- `api_base`: Base URL of your prompt management API
- `api_key`: Optional API key for authentication (sent as `Bearer` token)
- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
## Usage
### Using with LiteLLM SDK
**Basic usage with prompt ID:**
```python
from litellm import completion
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
messages=[{"role": "user", "content": "Additional message"}]
)
```
**With prompt variables:**
```python
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
prompt_variables={
"domain": "data science",
"task": "analyzing customer churn"
},
messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
)
```
The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
### Using with LiteLLM Proxy
**1. Start the proxy with your config:**
```bash
litellm --config /path/to/config.yaml
```
**2. Make requests with prompt_id:**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"prompt_id": "simple_prompt",
"prompt_variables": {
"domain": "healthcare",
"task": "patient risk assessment"
},
"messages": [
{"role": "user", "content": "Analyze the following data..."}
]
}'
```
**3. Using with OpenAI SDK:**
```python
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234"
)
response = client.chat.completions.create(
model="gpt-4",
messages=[
{"role": "user", "content": "Analyze the data"}
],
extra_body={
"prompt_id": "simple_prompt",
"prompt_variables": {
"domain": "finance",
"task": "fraud detection"
}
}
)
```
## Implementation Example
See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
**Minimal FastAPI example:**
```python
from fastapi import FastAPI, HTTPException, Header
from typing import Optional, Dict, Any, List
from pydantic import BaseModel
app = FastAPI()
# In-memory prompt storage (replace with your database)
PROMPTS = {
"hello-world-prompt": {
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
},
"code-review-prompt": {
"prompt_id": "code-review-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are an expert code reviewer. Review code for {language}."
},
{
"role": "user",
"content": "Review the following code:\n\n{code}"
}
],
"prompt_template_model": "gpt-4-turbo",
"prompt_template_optional_params": {
"temperature": 0.3,
"max_tokens": 1000
}
}
}
class PromptResponse(BaseModel):
prompt_id: str
prompt_template: List[Dict[str, str]]
prompt_template_model: Optional[str] = None
prompt_template_optional_params: Optional[Dict[str, Any]] = None
@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
async def get_prompt(
prompt_id: str,
authorization: Optional[str] = Header(None),
project_name: Optional[str] = None,
slug: Optional[str] = None,
):
"""
Get a prompt by ID with optional filtering by project_name and slug.
Args:
prompt_id: The ID of the prompt to fetch
authorization: Optional Bearer token for authentication
project_name: Optional project name filter
slug: Optional slug filter
"""
# Optional: Validate authorization
if authorization:
token = authorization.replace("Bearer ", "")
# Validate your token here
if not is_valid_token(token):
raise HTTPException(status_code=401, detail="Invalid API key")
# Optional: Apply additional filtering based on custom params
if project_name or slug:
# You can use these parameters to filter or validate access
# For example, check if the user has access to this project
pass
# Fetch the prompt from your storage
if prompt_id not in PROMPTS:
raise HTTPException(
status_code=404,
detail=f"Prompt '{prompt_id}' not found"
)
prompt_data = PROMPTS[prompt_id]
return PromptResponse(**prompt_data)
def is_valid_token(token: str) -> bool:
"""Validate API token - implement your logic here"""
# Example: Check against your database or secret store
valid_tokens = ["your-secret-token", "another-valid-token"]
return token in valid_tokens
# Optional: Health check endpoint
@app.get("/health")
async def health_check():
return {"status": "healthy"}
# Optional: List all prompts endpoint
@app.get("/prompts")
async def list_prompts(authorization: Optional[str] = Header(None)):
"""List all available prompts"""
if authorization:
token = authorization.replace("Bearer ", "")
if not is_valid_token(token):
raise HTTPException(status_code=401, detail="Invalid API key")
return {
"prompts": [
{"prompt_id": pid, "model": p.get("prompt_template_model")}
for pid, p in PROMPTS.items()
]
}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8080)
```
### Running the Example Server
1. Install dependencies:
```bash
pip install fastapi uvicorn
```
2. Save the code above to `prompt_server.py`
3. Run the server:
```bash
python prompt_server.py
```
4. Test the endpoint:
```bash
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
```
Expected response:
```json
{
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
## Advanced Features
### Variable Substitution
LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
**Example prompt template:**
```json
{
"prompt_template": [
{
"role": "system",
"content": "You are an expert in {domain} with {years} years of experience."
}
]
}
```
**Client request:**
```python
completion(
model="gpt-4",
prompt_id="expert_prompt",
prompt_variables={
"domain": "machine learning",
"years": "10"
}
)
```
**Result:**
```
"You are an expert in machine learning with 10 years of experience."
```
### Caching
LiteLLM automatically caches fetched prompts in memory. The cache key includes:
- `prompt_id`
- `prompt_label` (if provided)
- `prompt_version` (if provided)
This means your API endpoint is only called once per unique prompt configuration.
### Model Override Behavior
**Default behavior (without `ignore_prompt_manager_model`):**
```yaml
prompts:
- prompt_id: "my_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
```
If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
**With `ignore_prompt_manager_model: true`:**
```yaml
prompts:
- prompt_id: "my_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
ignore_prompt_manager_model: true
```
LiteLLM will use the model specified by the client, ignoring the prompt's model.
### Parameter Merging Behavior
**Default behavior (without `ignore_prompt_manager_optional_params`):**
Client params are merged with prompt params, with prompt params taking precedence:
```python
# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
# Client sends: {"temperature": 0.9, "top_p": 0.95}
# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
```
**With `ignore_prompt_manager_optional_params: true`:**
Only client params are used:
```python
# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
# Client sends: {"temperature": 0.9, "top_p": 0.95}
# Final params: {"temperature": 0.9, "top_p": 0.95}
```
## Security Considerations
1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
2. **Authorization**: Implement team/user-based access control using the custom query parameters
3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
4. **Input Validation**: Validate all query parameters before processing
5. **HTTPS**: Always use HTTPS in production for encrypted communication
6. **Secrets**: Store API keys in environment variables, not in config files
## Use Cases
✅ **Use Generic Prompt Management API when:**
- You want instant integration without waiting for PRs
- You maintain your own prompt management service
- You need full control over prompt versioning and updates
- You want to build custom prompt management features
- You need to integrate with your internal systems
✅ **Common scenarios:**
- Internal prompt management system for your organization
- Multi-tenant prompt management with team-based access control
- A/B testing different prompt versions
- Prompt experimentation and analytics
- Integration with existing prompt engineering workflows
## When to Use This
✅ **Use Generic Prompt Management API when:**
- You want instant integration without waiting for PRs
- You maintain your own prompt management service
- You need full control over updates and features
- You want custom prompt storage and versioning logic
❌ **Make a PR when:**
- You want deeper integration with LiteLLM internals
- Your integration requires complex LiteLLM-specific logic
- You want to be featured as a built-in provider
- You're building a reusable integration for the community
## Troubleshooting
### Prompt not found
- Verify the `prompt_id` matches exactly (case-sensitive)
- Check that your API endpoint is accessible from LiteLLM
- Verify authentication if using `api_key`
### Variables not substituted
- Ensure variables use `{variable}` or `{{variable}}` syntax
- Check that variable names in `prompt_variables` match template exactly
- Variables are case-sensitive
### Model not being overridden
- Check if `ignore_prompt_manager_model: true` is set in config
- Verify your API is returning `prompt_template_model` in the response
### Parameters not being applied
- Check if `ignore_prompt_manager_optional_params: true` is set
- Verify your API is returning `prompt_template_optional_params`
- Ensure parameter names match OpenAI's parameter names
## Questions?
This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
## Related Documentation
- [Prompt Management Overview](../proxy/prompt_management.md)
- [Generic Guardrail API](./generic_guardrail_api.md)
- [LiteLLM Proxy Setup](../proxy/quick_start.md)
docs/my-website/docs/completion/message_sanitization.md
+465
View File
@@ -0,0 +1,465 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Message Sanitization for Tool Calling for anthropic models
**Automatically fix common message formatting issues when using tool calling with `modify_params=True`**
LiteLLM can automatically sanitize messages to handle common issues that occur during tool calling workflows, especially when using OpenAI-compatible clients with providers that have strict message format requirements (like Anthropic Claude).
## Overview
When `litellm.modify_params = True` is enabled, LiteLLM automatically sanitizes messages to fix three common issues:
1. **Orphaned Tool Calls** - Assistant messages with tool_calls but missing tool results
2. **Orphaned Tool Results** - Tool messages that reference non-existent tool_call_ids
3. **Empty Message Content** - Messages with empty or whitespace-only text content
This ensures your tool calling workflows work seamlessly across different LLM providers without manual message validation.
## Why Message Sanitization?
Different LLM providers have varying requirements for message formats, especially during tool calling:
- **Anthropic Claude** requires every tool_call to have a corresponding tool result
- Some providers reject messages with empty content
- OpenAI-compatible clients may not always maintain perfect message consistency
Without sanitization, these issues cause API errors that interrupt your workflows. With `modify_params=True`, LiteLLM handles these edge cases automatically.
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Enable automatic message sanitization
litellm.modify_params = True
# This will work even if messages have formatting issues
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=[
{"role": "user", "content": "What's the weather in Boston?"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Boston"}'}
}
]
# Missing tool result - LiteLLM will add a dummy result automatically
},
{"role": "user", "content": "Thanks!"}
],
tools=[{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
litellm_settings:
modify_params: true # Enable automatic message sanitization
model_list:
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
```
</TabItem>
</Tabs>
## Sanitization Cases
### Case A: Orphaned Tool Calls (Missing Tool Results)
**Problem:** An assistant message contains `tool_calls`, but no corresponding tool result messages follow.
**Solution:** LiteLLM automatically adds dummy tool result messages for any missing tool results.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with orphaned tool calls
messages = [
{"role": "user", "content": "Search for Python tutorials"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "Python tutorials"}'}
}
]
},
# Missing tool result here!
{"role": "user", "content": "What about JavaScript?"}
]
# LiteLLM automatically adds:
# {
# "role": "tool",
# "tool_call_id": "call_abc123",
# "content": "[System: Tool execution skipped/interrupted by user. No result provided for tool 'web_search'.]"
# }
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
tools=[...]
)
```
**When this happens:**
- User interrupts tool execution
- Client loses tool results due to network issues
- Conversation flow changes before tool completes
- Multi-turn conversations where tools are optional
### Case B: Orphaned Tool Results (Invalid tool_call_id)
**Problem:** A tool message references a `tool_call_id` that doesn't exist in any previous assistant message.
**Solution:** LiteLLM automatically removes these orphaned tool result messages.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with orphaned tool result
messages = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi! How can I help?"},
{
"role": "tool",
"tool_call_id": "call_nonexistent", # This tool_call_id doesn't exist!
"content": "Some result"
}
]
# LiteLLM automatically removes the orphaned tool message
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
**When this happens:**
- Message history is manually edited
- Tool results are duplicated or mismatched
- Conversation state is restored incorrectly
- Messages are merged from different conversations
### Case C: Empty Message Content
**Problem:** User or assistant messages have empty or whitespace-only content.
**Solution:** LiteLLM replaces empty content with a system placeholder message.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with empty content
messages = [
{"role": "user", "content": ""}, # Empty content
{"role": "assistant", "content": " "}, # Whitespace only
]
# LiteLLM automatically replaces with:
# {"role": "user", "content": "[System: Empty message content sanitised to satisfy protocol]"}
# {"role": "assistant", "content": "[System: Empty message content sanitised to satisfy protocol]"}
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
**When this happens:**
- UI sends empty messages
- Content is stripped during preprocessing
- Placeholder messages in conversation history
- Edge cases in message construction
## Configuration
### Enable Globally
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Enable for all completion calls
litellm.modify_params = True
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
litellm_settings:
modify_params: true
```
</TabItem>
<TabItem value="env" label="Environment Variable">
```bash
export LITELLM_MODIFY_PARAMS=True
```
</TabItem>
</Tabs>
### Enable Per-Request
```python
import litellm
# Enable only for specific requests
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
modify_params=True # Override global setting
)
```
## Supported Providers
Message sanitization currently works with:
- ✅ Anthropic (Claude)
**Note:** While the sanitization logic is provider-agnostic, it is currently only applied in the Anthropic message transformation pipeline. Support for additional providers may be added in future releases.
## Implementation Details
### How It Works
The message sanitization process runs **before** messages are converted to provider-specific formats:
1. **Input:** OpenAI-format messages with potential issues
2. **Sanitization:** Three helper functions process the messages:
- `_sanitize_empty_text_content()` - Fixes empty content
- `_add_missing_tool_results()` - Adds dummy tool results
- `_is_orphaned_tool_result()` - Identifies orphaned results
3. **Output:** Clean, provider-compatible messages
### Code Reference
The sanitization logic is implemented in:
- `litellm/litellm_core_utils/prompt_templates/factory.py`
- Function: `sanitize_messages_for_tool_calling()`
### Logging
When sanitization occurs, LiteLLM logs debug messages:
```python
import litellm
litellm.set_verbose = True # Enable debug logging
# You'll see logs like:
# "_add_missing_tool_results: Found 1 orphaned tool calls. Adding dummy tool results."
# "_is_orphaned_tool_result: Found orphaned tool result with tool_call_id=call_123"
# "_sanitize_empty_text_content: Replaced empty text content in user message"
```
## Best Practices
### 1. Enable for Production Workflows
```python
# Recommended for production
litellm.modify_params = True
# Ensures robust handling of edge cases
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
tools=tools
)
```
### 2. Preserve Tool Results When Possible
While sanitization handles missing tool results, it's better to provide actual results:
```python
# Good: Provide actual tool results
messages = [
{"role": "user", "content": "Search for Python"},
{"role": "assistant", "tool_calls": [...]},
{"role": "tool", "tool_call_id": "call_123", "content": "Actual search results"}
]
# Fallback: Sanitization adds dummy result if missing
messages = [
{"role": "user", "content": "Search for Python"},
{"role": "assistant", "tool_calls": [...]},
# Missing tool result - sanitization adds dummy
]
```
### 3. Monitor Sanitization Events
Use logging to track when sanitization occurs:
```python
import litellm
import logging
# Enable debug logging
litellm.set_verbose = True
logging.basicConfig(level=logging.DEBUG)
# Track sanitization events in your application
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
### 4. Test Edge Cases
Ensure your application handles sanitized messages correctly:
```python
import litellm
litellm.modify_params = True
# Test orphaned tool calls
test_messages = [
{"role": "user", "content": "Test"},
{"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "test", "arguments": "{}"}}]},
{"role": "user", "content": "Continue"} # No tool result
]
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=test_messages,
tools=[...]
)
# Verify the response handles the dummy tool result appropriately
```
## Related Features
- **[Drop Params](./drop_params.md)** - Drop unsupported parameters for specific providers
- **[Message Trimming](./message_trimming.md)** - Trim messages to fit token limits
- **[Function Calling](./function_call.md)** - Complete guide to tool/function calling
- **[Reasoning Content](../reasoning_content.md)** - Extended thinking with tool calling
## Troubleshooting
### Sanitization Not Working
**Issue:** Messages still cause errors despite `modify_params=True`
**Solution:**
1. Verify `modify_params` is enabled:
```python
import litellm
print(litellm.modify_params) # Should be True
```
2. Check if the issue is provider-specific:
```python
litellm.set_verbose = True # Enable debug logging
```
3. Ensure you're using a recent version of LiteLLM:
```bash
pip install --upgrade litellm
```
### Unexpected Dummy Tool Results
**Issue:** Dummy tool results appear when you expect actual results
**Cause:** Tool result messages are missing or have incorrect `tool_call_id`
**Solution:**
1. Verify tool result messages have correct `tool_call_id`:
```python
# Correct
{"role": "tool", "tool_call_id": "call_123", "content": "result"}
# Incorrect - will be treated as orphaned
{"role": "tool", "tool_call_id": "wrong_id", "content": "result"}
```
2. Ensure tool results immediately follow assistant messages with tool_calls
### Performance Impact
**Issue:** Concerned about performance overhead
**Details:** Message sanitization has minimal performance impact:
- Runs in O(n) time where n = number of messages
- Only processes messages when `modify_params=True`
- Typically adds < 1ms to request processing time
## FAQ
**Q: Does sanitization modify my original messages?**
A: No, sanitization creates a new list of messages. Your original messages remain unchanged.
**Q: Can I disable specific sanitization cases?**
A: Currently, all three cases are handled together when `modify_params=True`. To disable sanitization entirely, set `modify_params=False`.
**Q: What happens to the dummy tool results?**
A: Dummy tool results are sent to the LLM provider along with other messages. The model sees them as regular tool results with informative error messages.
**Q: Does this work with streaming?**
A: Yes, message sanitization works with both streaming and non-streaming requests.
**Q: Is this related to `drop_params`?**
A: No, they're separate features:
- `modify_params` - Modifies/fixes message content and structure
- `drop_params` - Removes unsupported API parameters
Both can be enabled simultaneously.
## See Also
- [Reasoning Content with Tool Calling](../reasoning_content.md)
- [Function Calling Guide](./function_call.md)
- [Bedrock Provider Documentation](../providers/bedrock.md)
- [Anthropic Provider Documentation](../providers/anthropic.md)
docs/my-website/docs/completion/usage.md
+48
View File
@@ -50,3 +50,51 @@ for chunk in completion:
print(chunk.choices[0].delta)
```
### Proxy: Always Include Streaming Usage
When using the LiteLLM Proxy, you can configure it to automatically include usage information in all streaming responses, even if the client doesn't send `stream_options={"include_usage": True}`.
#### Configuration
Add the following to your config.yaml:
```yaml
general_settings:
always_include_stream_usage: true
```
Alternatively, configure it through the UI:
1. Navigate to the LiteLLM Proxy UI
2. Go to `Settings` > `Router Settings` > `General`
3. Find the `always_include_stream_usage` setting
4. Toggle it to `true`
5. Click `Update` to save
#### How it works
When `always_include_stream_usage` is enabled:
- All streaming requests will automatically have `stream_options={"include_usage": True}` added
- Clients will receive usage information in the final chunk, even if they didn't explicitly request it
- If a client already provides `stream_options`, `include_usage: True` will be added without overwriting other options
- Non-streaming requests are not affected
#### Example
With this setting enabled, a simple streaming request like:
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello!"}],
"stream": true
}'
```
Will automatically receive usage information in the response, without needing to explicitly include `stream_options`.
```
docs/my-website/docs/completion/web_search.md
+78 -39
View File
@@ -18,7 +18,7 @@ Each provider uses their own search backend:
| Provider | Search Engine | Notes |
|----------|---------------|-------|
| **OpenAI** (`gpt-4o-search-preview`, `gpt-4o-mini-search-preview`, `gpt-5-search-api`) | OpenAI's internal search | Real-time web data |
| **OpenAI** (`gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview`) | OpenAI's internal search | Real-time web data |
| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
@@ -45,6 +45,19 @@ Use `web_search_options` when you need to:
**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
:::
## OpenAI Web Search: Two Approaches
OpenAI offers two distinct ways to use web search depending on the endpoint and model:
| Approach | Endpoint | Models | How to enable |
|----------|----------|--------|---------------|
| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
:::tip Search models search automatically
Search models like `gpt-5-search-api` **automatically search the web** even without the `web_search_options` parameter. Use `web_search_options` to set `search_context_size` (`"low"`, `"medium"`, `"high"`) or specify `user_location` for localized results.
:::
## `/chat/completions` (litellm.completion)
### Quick Start
@@ -56,7 +69,7 @@ Use `web_search_options` when you need to:
from litellm import completion
response = completion(
model="openai/gpt-4o-search-preview",
model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@@ -76,31 +89,36 @@ response = completion(
```yaml
model_list:
# OpenAI
# OpenAI search models
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
# xAI
- model_name: grok-3
litellm_params:
model: xai/grok-3
api_key: os.environ/XAI_API_KEY
# Anthropic
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
# VertexAI
- model_name: gemini-2-flash
litellm_params:
model: gemini-2.0-flash
vertex_project: your-project-id
vertex_location: us-central1
# Google AI Studio
- model_name: gemini-2-flash-studio
litellm_params:
@@ -108,13 +126,13 @@ model_list:
api_key: os.environ/GOOGLE_API_KEY
```
2. Start the proxy
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python showLineNumbers
from openai import OpenAI
@@ -126,13 +144,18 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="grok-3", # or any other web search enabled model
model="gpt-5-search-api", # or any other web search enabled model
messages=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
]
],
extra_body={
"web_search_options": {
"search_context_size": "medium"
}
}
)
```
</TabItem>
@@ -149,7 +172,7 @@ from litellm import completion
# Customize search context size
response = completion(
model="openai/gpt-4o-search-preview",
model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@@ -257,6 +280,12 @@ response = client.chat.completions.create(
## `/responses` (litellm.responses)
Use the `web_search_preview` tool with models like `gpt-5`, `gpt-4.1`, `gpt-4o`, etc.
:::info
Search-dedicated models like `gpt-5-search-api` and `gpt-4o-search-preview` do **not** support the `/responses` endpoint. Use them with `/chat/completions` + `web_search_options` instead (see above).
:::
### Quick Start
<Tabs>
@@ -266,18 +295,14 @@ response = client.chat.completions.create(
from litellm import responses
response = responses(
model="openai/gpt-4o",
input=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
],
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview" # enables web search with default medium context size
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
@@ -285,19 +310,24 @@ response = responses(
```yaml
model_list:
- model_name: gpt-4o
- model_name: gpt-5
litellm_params:
model: openai/gpt-4o
model: openai/gpt-5
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4.1
litellm_params:
model: openai/gpt-4.1
api_key: os.environ/OPENAI_API_KEY
```
2. Start the proxy
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python showLineNumbers
from openai import OpenAI
@@ -309,11 +339,11 @@ client = OpenAI(
)
response = client.responses.create(
model="gpt-4o",
model="gpt-5",
tools=[{
"type": "web_search_preview"
}],
input="What was a positive news story from today?",
input="What is the capital of France?",
)
print(response.output_text)
@@ -331,13 +361,8 @@ from litellm import responses
# Customize search context size
response = responses(
model="openai/gpt-4o",
input=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
],
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
@@ -358,12 +383,12 @@ client = OpenAI(
# Customize search context size
response = client.responses.create(
model="gpt-4o",
model="gpt-5",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
}],
input="What was a positive news story from today?",
input="What is the capital of France?",
)
print(response.output_text)
@@ -417,14 +442,14 @@ model_list:
web_search_options:
search_context_size: "high" # Options: "low", "medium", "high"
# Different context size for different models
- model_name: gpt-4o-search-preview
# OpenAI search model with custom context size
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-4o-search-preview
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
web_search_options:
search_context_size: "low"
# Gemini with medium context (default)
- model_name: gemini-2-flash
litellm_params:
@@ -449,6 +474,7 @@ Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model
```python showLineNumbers
# Check OpenAI models
assert litellm.supports_web_search(model="openai/gpt-5-search-api") == True
assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
# Check xAI models
@@ -472,13 +498,20 @@ assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
```yaml
model_list:
# OpenAI
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
# xAI
- model_name: grok-3
litellm_params:
@@ -533,6 +566,12 @@ Expected Response
```json showLineNumbers
{
"data": [
{
"model_group": "gpt-5-search-api",
"providers": ["openai"],
"max_tokens": 128000,
"supports_web_search": true
},
{
"model_group": "gpt-4o-search-preview",
"providers": ["openai"],
docs/my-website/docs/evals_api.md
+441
View File
@@ -0,0 +1,441 @@
# /evals
LiteLLM Proxy supports OpenAI's Evaluations (Evals) API, allowing you to create, manage, and run evaluations to measure model performance against defined testing criteria.
## What are Evals?
OpenAI Evals API provides a structured way to:
- **Create Evaluations**: Define testing criteria and data sources for evaluating model outputs
- **Run Evaluations**: Execute evaluations against specific models and datasets
- **Track Results**: Monitor evaluation progress and review detailed results
## Quick Start
### Setup LiteLLM Proxy
First, start your LiteLLM Proxy server:
```bash
litellm --config config.yaml
# Proxy will run on http://localhost:4000
```
### Initialize OpenAI Client
```python
from openai import OpenAI
# Point to your LiteLLM Proxy
client = OpenAI(
api_key="sk-1234", # Your LiteLLM proxy API key
base_url="http://localhost:4000" # Your proxy URL
)
```
For async operations:
```python
from openai import AsyncOpenAI
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
```
---
## Evaluation Management
### Create an Evaluation
Create an evaluation with testing criteria and data source configuration.
#### Example: Sentiment Classification Eval
```python
from openai import OpenAI
client = OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Create evaluation with label model grader
eval_obj = client.evals.create(
name="Sentiment Classification",
data_source_config={
"type": "stored_completions",
"metadata": {"usecase": "chatbot"}
},
testing_criteria=[
{
"type": "label_model",
"model": "gpt-4o-mini",
"input": [
{
"role": "developer",
"content": "Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'"
},
{
"role": "user",
"content": "Statement: {{item.input}}"
}
],
"passing_labels": ["positive"],
"labels": ["positive", "neutral", "negative"],
"name": "Sentiment Grader"
}
]
)
# Note: If you want to use model-specific credentials for this evaluation, you can specify the model name in the extra body parameters.
print(f"Created eval: {eval_obj.id}")
print(f"Eval name: {eval_obj.name}")
```
#### Example: Push Notifications Summarizer Monitoring
This example shows how to monitor prompt changes for regressions in a push notifications summarizer:
```python
from openai import AsyncOpenAI
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Define data source for stored completions
data_source_config = {
"type": "stored_completions",
"metadata": {
"usecase": "push_notifications_summarizer"
}
}
# Define grader criteria
GRADER_DEVELOPER_PROMPT = """
Label the following push notification summary as either correct or incorrect.
The push notification and the summary will be provided below.
A good push notification summary is concise and snappy.
If it is good, then label it as correct, if not, then incorrect.
"""
GRADER_TEMPLATE_PROMPT = """
Push notifications: {{item.input}}
Summary: {{sample.output_text}}
"""
push_notification_grader = {
"name": "Push Notification Summary Grader",
"type": "label_model",
"model": "gpt-4o-mini",
"input": [
{
"role": "developer",
"content": GRADER_DEVELOPER_PROMPT,
},
{
"role": "user",
"content": GRADER_TEMPLATE_PROMPT,
},
],
"passing_labels": ["correct"],
"labels": ["correct", "incorrect"],
}
# Create the evaluation
eval_result = await client.evals.create(
name="Push Notification Completion Monitoring",
metadata={"description": "This eval monitors completions"},
data_source_config=data_source_config,
testing_criteria=[push_notification_grader],
)
eval_id = eval_result.id
print(f"Created eval: {eval_id}")
```
### List Evaluations
Retrieve a list of all your evaluations with pagination support.
```python
# List all evaluations
evals_response = client.evals.list(
limit=20,
order="desc"
)
for eval in evals_response.data:
print(f"Eval ID: {eval.id}, Name: {eval.name}")
# Check if there are more evals
if evals_response.has_more:
# Fetch next page
next_evals = client.evals.list(
after=evals_response.last_id,
limit=20
)
```
### Get a Specific Evaluation
Retrieve details of a specific evaluation by ID.
```python
eval = client.evals.retrieve(
eval_id="eval_abc123"
)
print(f"Eval ID: {eval.id}")
print(f"Name: {eval.name}")
print(f"Data Source: {eval.data_source_config}")
print(f"Testing Criteria: {eval.testing_criteria}")
```
### Update an Evaluation
Update evaluation metadata or name.
```python
updated_eval = client.evals.update(
eval_id="eval_abc123",
name="Updated Evaluation Name",
metadata={
"version": "2.0",
"updated_by": "user@example.com"
}
)
print(f"Updated eval: {updated_eval.name}")
```
### Delete an Evaluation
Permanently delete an evaluation.
```python
delete_response = client.evals.delete(
eval_id="eval_abc123"
)
print(f"Deleted: {delete_response.deleted}") # True
```
---
## Evaluation Runs
### Create a Run
Execute an evaluation by creating a run. The run processes your data through the model and applies testing criteria.
#### Using Stored Completions
First, generate some test data by making chat completions with metadata:
```python
from openai import AsyncOpenAI
import asyncio
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Generate test data with different prompt versions
push_notification_data = [
"""
- New message from Sarah: "Can you call me later?"
- Your package has been delivered!
- Flash sale: 20% off electronics for the next 2 hours!
""",
"""
- Weather alert: Thunderstorm expected in your area.
- Reminder: Doctor's appointment at 3 PM.
- John liked your photo on Instagram.
"""
]
PROMPTS = [
(
"""
You are a helpful assistant that summarizes push notifications.
You are given a list of push notifications and you need to collapse them into a single one.
Output only the final summary, nothing else.
""",
"v1"
),
(
"""
You are a helpful assistant that summarizes push notifications.
You are given a list of push notifications and you need to collapse them into a single one.
The summary should be longer than it needs to be and include more information than is necessary.
Output only the final summary, nothing else.
""",
"v2"
)
]
# Create completions with metadata for tracking
tasks = []
for notifications in push_notification_data:
for (prompt, version) in PROMPTS:
tasks.append(client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "developer", "content": prompt},
{"role": "user", "content": notifications},
],
metadata={
"prompt_version": version,
"usecase": "push_notifications_summarizer"
}
))
await asyncio.gather(*tasks)
```
Now create runs to evaluate different prompt versions:
```python
# Grade prompt_version=v1
eval_run_result = await client.evals.runs.create(
eval_id=eval_id,
name="v1-run",
data_source={
"type": "completions",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": "v1",
}
}
}
)
print(f"Run ID: {eval_run_result.id}")
print(f"Status: {eval_run_result.status}")
print(f"Report URL: {eval_run_result.report_url}")
# Grade prompt_version=v2
eval_run_result_v2 = await client.evals.runs.create(
eval_id=eval_id,
name="v2-run",
data_source={
"type": "completions",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": "v2",
}
}
}
)
print(f"Run ID: {eval_run_result_v2.id}")
print(f"Report URL: {eval_run_result_v2.report_url}")
```
#### Using Completions with Different Models
Test how different models perform on the same inputs:
```python
# Test with GPT-4o using stored completions as input
tasks = []
for prompt_version in ["v1", "v2"]:
tasks.append(client.evals.runs.create(
eval_id=eval_id,
name=f"gpt-4o-run-{prompt_version}",
data_source={
"type": "completions",
"input_messages": {
"type": "item_reference",
"item_reference": "item.input",
},
"model": "gpt-4o",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": prompt_version,
}
}
}
))
results = await asyncio.gather(*tasks)
for run in results:
print(f"Report URL: {run.report_url}")
```
### List Runs
Get all runs for a specific evaluation.
```python
# List all runs for an evaluation
runs_response = client.evals.runs.list(
eval_id="eval_abc123",
limit=20,
order="desc"
)
for run in runs_response.data:
print(f"Run ID: {run.id}")
print(f"Status: {run.status}")
print(f"Name: {run.name}")
if run.result_counts:
print(f"Results: {run.result_counts.passed}/{run.result_counts.total} passed")
```
### Get Run Details
Retrieve detailed information about a specific run, including results.
```python
run = client.evals.runs.retrieve(
eval_id="eval_abc123",
run_id="run_def456"
)
print(f"Run ID: {run.id}")
print(f"Status: {run.status}")
print(f"Started: {run.started_at}")
print(f"Completed: {run.completed_at}")
# Check results
if run.result_counts:
print(f"\nOverall Results:")
print(f"Total: {run.result_counts.total}")
print(f"Passed: {run.result_counts.passed}")
print(f"Failed: {run.result_counts.failed}")
print(f"Error: {run.result_counts.errored}")
# Per-criteria results
if run.per_testing_criteria_results:
for criteria_result in run.per_testing_criteria_results:
print(f"\nCriteria {criteria_result.testing_criteria_index}:")
print(f" Passed: {criteria_result.result_counts.passed}")
print(f" Average Score: {criteria_result.average_score}")
```
### Delete a Run
Permanently delete a run and its results.
```python
delete_response = await client.evals.runs.delete(
eval_id="eval_abc123",
run_id="run_def456"
)
print(f"Deleted: {delete_response.deleted}") # True
print(f"Run ID: {delete_response.run_id}")
```
docs/my-website/docs/mcp.md
+62
View File
@@ -808,6 +808,68 @@ If your stdio MCP server needs per-request credentials, you can map HTTP headers
In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
## Control MCP Access for End Users
Control which MCP servers end users of your AI application can access (e.g. users of an internal chat UI). Pass the customer ID in the `x-litellm-end-user-id` header to:
- Enforce object permissions (limit which MCP servers they can access)
- Apply customer-specific budgets
- Track spend per customer
**FastMCP Client Example:**
```python title="Track customer spend with x-litellm-end-user-id" showLineNumbers
from fastmcp import Client
import asyncio
# MCP client configuration with customer tracking
config = {
"mcpServers": {
"github": {
"url": "http://localhost:4000/github_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234",
"x-litellm-end-user-id": "customer_123", # 👈 CUSTOMER ID
"Authorization": "Bearer gho_token"
}
}
}
}
client = Client(config)
async def main():
async with client:
# All MCP calls will be tracked under customer_123
tools = await client.list_tools()
result = await client.call_tool(tools[0].name, {})
print(f"Tool result: {result}")
asyncio.run(main())
```
**Cursor IDE Example:**
```json title="Cursor config with customer tracking" showLineNumbers
{
"mcpServers": {
"GitHub": {
"url": "http://localhost:4000/github_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY",
"x-litellm-end-user-id": "customer_123"
}
}
}
}
```
**What happens:**
- Customer-specific object permissions are enforced (only allowed MCP servers are accessible)
- Customer budgets are applied
- All tool calls are tracked under `customer_123`
[Learn more about customer management →](./proxy/customers)
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
docs/my-website/docs/mcp_oauth.md
+93
View File
@@ -242,3 +242,96 @@ curl http://localhost:4000/mcp-rest/tools/call \
| `client_secret` | Yes | OAuth2 client secret. Supports `os.environ/VAR_NAME` |
| `token_url` | Yes | Token endpoint URL |
| `scopes` | No | List of scopes to request |
## Debugging OAuth
When the LiteLLM proxy is hosted remotely and you cannot access server logs, enable **debug headers** to get masked authentication diagnostics in the HTTP response.
### Enable Debug Mode
Add the `x-litellm-mcp-debug: true` header to your MCP client request.
**Claude Code:**
```bash
claude mcp add --transport http litellm_proxy http://proxy.example.com/atlassian_mcp/mcp \
--header "x-litellm-api-key: Bearer sk-..." \
--header "x-litellm-mcp-debug: true"
```
**curl:**
```bash
curl -X POST http://localhost:4000/atlassian_mcp/mcp \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-..." \
-H "x-litellm-mcp-debug: true" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
### Reading the Debug Response Headers
The response includes these headers (all sensitive values are masked):
| Header | Description |
|--------|-------------|
| `x-mcp-debug-inbound-auth` | Which inbound auth headers were present. |
| `x-mcp-debug-oauth2-token` | The OAuth2 token (masked). Shows `SAME_AS_LITELLM_KEY` if the LiteLLM key is leaking. |
| `x-mcp-debug-auth-resolution` | Which auth method was used: `oauth2-passthrough`, `m2m-client-credentials`, `per-request-header`, `static-token`, or `no-auth`. |
| `x-mcp-debug-outbound-url` | The upstream MCP server URL. |
| `x-mcp-debug-server-auth-type` | The `auth_type` configured on the server. |
**Example — healthy OAuth2 passthrough:**
```
x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234; authorization=Bearer****ef01
x-mcp-debug-oauth2-token: Bearer****ef01
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
**Example — LiteLLM key leaking (misconfigured):**
```
x-mcp-debug-inbound-auth: authorization=Bearer****1234
x-mcp-debug-oauth2-token: Bearer****1234 (SAME_AS_LITELLM_KEY - likely misconfigured)
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
### Common Issues
#### LiteLLM API key leaking to the MCP server
**Symptom:** `x-mcp-debug-oauth2-token` shows `SAME_AS_LITELLM_KEY`.
The `Authorization` header carries the LiteLLM API key instead of an OAuth2 token. The OAuth2 flow never ran because the client already had an `Authorization` header set.
**Fix:** Move the LiteLLM key to `x-litellm-api-key`:
```bash
# WRONG — blocks OAuth2 discovery
claude mcp add --transport http my_server http://proxy/mcp/server \
--header "Authorization: Bearer sk-..."
# CORRECT — LiteLLM key in dedicated header, Authorization free for OAuth2
claude mcp add --transport http my_server http://proxy/mcp/server \
--header "x-litellm-api-key: Bearer sk-..."
```
#### No OAuth2 token present
**Symptom:** `x-mcp-debug-oauth2-token` shows `(none)` and `x-mcp-debug-auth-resolution` shows `no-auth`.
Check that:
1. The `Authorization` header is NOT set as a static header in the client config.
2. The MCP server in LiteLLM config has `auth_type: oauth2`.
3. The `.well-known/oauth-protected-resource` endpoint returns valid metadata.
#### M2M token used instead of user token
**Symptom:** `x-mcp-debug-auth-resolution` shows `m2m-client-credentials`.
The server has `client_id`/`client_secret`/`token_url` configured so LiteLLM is fetching a machine-to-machine token instead of using the per-user OAuth2 token. To use per-user tokens, remove the client credentials from the server config.
docs/my-website/docs/mcp_troubleshoot.md
+42 -5
View File
@@ -6,6 +6,39 @@ When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Pr
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
## Quick Start: Debug with One Command
The fastest way to debug MCP issues is to enable **debug headers**. Run this curl against your LiteLLM proxy and check the response headers:
```bash
curl -si -X POST http://localhost:4000/{your_mcp_server}/mcp \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-YOUR_KEY" \
-H "x-litellm-mcp-debug: true" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \
2>&1 | grep -i "x-mcp-debug"
```
This returns masked diagnostic headers that tell you exactly what's happening with authentication:
```
x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234
x-mcp-debug-oauth2-token: Bearer****ef01
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
If you see `SAME_AS_LITELLM_KEY` in `x-mcp-debug-oauth2-token`, your LiteLLM API key is leaking to the MCP server instead of an OAuth2 token. See [Debugging OAuth](./mcp_oauth#debugging-oauth) for the fix and other common issues.
For Claude Code, add the debug header to your MCP config:
```bash
claude mcp add --transport http my_server http://localhost:4000/my_mcp/mcp \
--header "x-litellm-api-key: Bearer sk-..." \
--header "x-litellm-mcp-debug: true"
```
## Locate the Error Source
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
@@ -13,7 +46,7 @@ Pin down where the failure occurs before adjusting settings so you do not mix sy
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
<Image
<Image
img={require('../img/mcp_tool_testing_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
@@ -22,7 +55,7 @@ Failures shown on the MCP creation form or within the MCP Tool Testing Playgroun
**Actions**
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
- From the LiteLLM server, run a [`curl` smoke test](./mcp_troubleshoot#curl-smoke-test) against the MCP endpoint to confirm basic connectivity.
### Client Traffic Issues (Client → LiteLLM)
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
@@ -43,7 +76,7 @@ During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls m
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
<Image
<Image
img={require('../img/mcp_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
@@ -55,6 +88,10 @@ LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://mode
- Use `curl <metadata_url>` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
## Debugging OAuth
For detailed OAuth2 debugging — including debug header reference, common misconfigurations, and example output — see [Debugging OAuth](./mcp_oauth#debugging-oauth).
## Verify Connectivity
Run lightweight validations before impacting production traffic.
@@ -66,7 +103,7 @@ Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Clien
2. Configure and connect:
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
- **Custom Headers:** e.g., `Authorization: Bearer <LiteLLM API Key>`.
- **Custom Headers:** e.g., `x-litellm-api-key: Bearer <LiteLLM API Key>`.
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
### `curl` Smoke Test
@@ -79,7 +116,7 @@ curl -X POST https://your-target-domain.example.com/mcp \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
Add `-H "Authorization: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
Add `-H "x-litellm-api-key: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
## Review Logs
docs/my-website/docs/observability/datadog.md
+9
View File
@@ -253,3 +253,12 @@ LiteLLM supports customizing the following Datadog environment variables
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)
## Automatic Tags
LiteLLM automatically adds the following tags to your Datadog logs and metrics if the information is available in the request:
| Tag | Description | Source |
|-----|-------------|--------|
| `team` | The team alias or ID associated with the API Key | `user_api_key_team_alias`, `team_alias`, `user_api_key_team_id`, or `team_id` in metadata |
| `request_tag` | Custom tags passed in the request | `request_tags` in logging payload |
docs/my-website/docs/projects/openai-agents.md
+106 -7
View File
@@ -1,22 +1,121 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# OpenAI Agents SDK
The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows.
It includes an official LiteLLM extension that lets you use any of the 100+ supported providers (Anthropic, Gemini, Mistral, Bedrock, etc.)
Use OpenAI Agents SDK with any LLM provider through LiteLLM Proxy.
The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows. It includes an official LiteLLM extension that lets you use any of the 100+ supported providers.
## Quick Start
### 1. Install Dependencies
```bash
pip install "openai-agents[litellm]"
```
### 2. Add Model to Config
```yaml title="config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: "openai/gpt-4o"
api_key: "os.environ/OPENAI_API_KEY"
- model_name: claude-sonnet
litellm_params:
model: "anthropic/claude-3-5-sonnet-20241022"
api_key: "os.environ/ANTHROPIC_API_KEY"
- model_name: gemini-pro
litellm_params:
model: "gemini/gemini-2.0-flash-exp"
api_key: "os.environ/GEMINI_API_KEY"
```
### 3. Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### 4. Use with Proxy
<Tabs>
<TabItem value="proxy" label="Via Proxy">
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
# Point to LiteLLM proxy
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
model=LitellmModel(model="provider/model-name")
model=LitellmModel(
model="claude-sonnet", # Model from config.yaml
api_key="sk-1234", # LiteLLM API key
base_url="http://localhost:4000"
)
)
result = Runner.run_sync(agent, "your_prompt_here")
print("Result:", result.final_output)
result = await Runner.run(agent, "What is LiteLLM?")
print(result.final_output)
```
- [GitHub](https://github.com/openai/openai-agents-python)
- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/ref/extensions/litellm/)
</TabItem>
<TabItem value="direct" label="Direct (No Proxy)">
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
# Use any provider directly
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
model=LitellmModel(
model="anthropic/claude-3-5-sonnet-20241022",
api_key="your-anthropic-key"
)
)
result = await Runner.run(agent, "What is LiteLLM?")
print(result.final_output)
```
</TabItem>
</Tabs>
## Track Usage
Enable usage tracking to monitor token consumption:
```python
from agents import Agent, ModelSettings
from agents.extensions.models.litellm_model import LitellmModel
agent = Agent(
name="Assistant",
model=LitellmModel(model="claude-sonnet", api_key="sk-1234"),
model_settings=ModelSettings(include_usage=True)
)
result = await Runner.run(agent, "Hello")
print(result.context_wrapper.usage) # Token counts
```
## Environment Variables
| Variable | Value | Description |
|----------|-------|-------------|
| `LITELLM_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
| `LITELLM_API_KEY` | `sk-1234` | Your LiteLLM API key |
## Related Resources
- [OpenAI Agents SDK Documentation](https://openai.github.io/openai-agents-python/)
- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/models/litellm/)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)
docs/my-website/docs/providers/anthropic.md
+71
View File
@@ -1473,6 +1473,20 @@ LiteLLM translates OpenAI's `reasoning_effort` to Anthropic's `thinking` paramet
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
:::note
For Claude Opus 4.6, all `reasoning_effort` values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets, pass the native `thinking` parameter directly:
```python
from litellm import completion
resp = completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the capital of France?"}],
thinking={"type": "enabled", "budget_tokens": 1024},
)
```
:::
<Tabs>
<TabItem value="sdk" label="SDK">
@@ -1614,8 +1628,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Adaptive Thinking (Claude Opus 4.6)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
thinking={"type": "adaptive"},
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "anthropic/claude-opus-4-6",
"messages": [{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
"thinking": {"type": "adaptive"}
}'
```
</TabItem>
</Tabs>
#### Enabled Thinking with Budget
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the capital of France?"}],
thinking={"type": "enabled", "budget_tokens": 5000},
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "anthropic/claude-opus-4-6",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"thinking": {"type": "enabled", "budget_tokens": 5000}
}'
```
</TabItem>
</Tabs>
## **Passing Extra Headers to Anthropic API**
docs/my-website/docs/providers/dashscope.md
+23 -5
View File
@@ -1,7 +1,7 @@
# Dashscope (Qwen API)
# Dashscope API (Qwen models)
https://dashscope.console.aliyun.com/
**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**
**We support ALL Qwen models (from Alibaba Cloud), just set `dashscope/` as a prefix when sending completion requests**
## API Key
```python
@@ -9,6 +9,26 @@ https://dashscope.console.aliyun.com/
os.environ['DASHSCOPE_API_KEY']
```
## API Base
You can optionally specify the API base URL depending on your region:
| Region | API Base |
|--------|----------|
| **International** | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` |
| **China/Beijing** | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
```python
# Set via environment variable
os.environ['DASHSCOPE_API_BASE'] = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
# Or pass directly in the completion call
response = completion(
model="dashscope/qwen-turbo",
messages=[{"role": "user", "content": "hello"}],
api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
)
```
## Sample Usage
```python
from litellm import completion
@@ -43,9 +63,7 @@ for chunk in response:
```
## Supported Models - ALL Qwen Models Supported!
We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests
## All supported Models
[DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz)
docs/my-website/docs/providers/gemini.md
+2
View File
@@ -1196,6 +1196,8 @@ When responding to Computer Use tool calls, include the URL and screenshot:
## Thought Signatures
Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry.
docs/my-website/docs/providers/openai.md
+64 -1
View File
@@ -230,7 +230,70 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
These also support the `OPENAI_BASE_URL` environment variable, which can be used to specify a custom API endpoint.
## OpenAI Vision Models
### OpenAI Web Search Models
OpenAI has two ways to use web search, depending on the endpoint:
| Approach | Endpoint | Models | How to enable |
|----------|----------|--------|---------------|
| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
<Tabs>
<TabItem value="sdk-completion" label="SDK - /chat/completions">
```python showLineNumbers
from litellm import completion
response = completion(
model="openai/gpt-5-search-api",
messages=[{"role": "user", "content": "What is the capital of France?"}],
web_search_options={
"search_context_size": "medium" # Options: "low", "medium", "high"
}
)
```
</TabItem>
<TabItem value="sdk-responses" label="SDK - /responses">
```python showLineNumbers
from litellm import responses
response = responses(
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "low"
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
# Search model for /chat/completions
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
# Regular model for /responses with web_search_preview tool
- model_name: gpt-5
litellm_params:
model: openai/gpt-5
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
</Tabs>
For full details, see the [Web Search guide](../completion/web_search.md).
## OpenAI Vision Models
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
| gpt-4o | `response = completion(model="gpt-4o", messages=messages)` |
docs/my-website/docs/providers/openai/responses_api.md
+18
View File
@@ -37,6 +37,24 @@ for event in response:
print(event)
```
#### Web Search
```python showLineNumbers title="OpenAI Responses with Web Search"
import litellm
response = litellm.responses(
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "medium" # Options: "low", "medium", "high"
}]
)
print(response)
```
For full details, see the [Web Search guide](../../completion/web_search.md).
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Streaming Image Generation"
import litellm
docs/my-website/docs/providers/scaleway.md
+62
View File
@@ -0,0 +1,62 @@
# Scaleway
LiteLLM supports all [models available on Scaleway Generative APIs ↗](https://www.scaleway.com/en/docs/generative-apis/reference-content/supported-models/).
## Usage with LiteLLM Python SDK
```python
import os
from litellm import completion
os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key"
messages = [{"role": "user", "content": "Write a short poem"}]
response = completion(model="scaleway/qwen3-235b-a22b-instruct-2507", messages=messages)
print(response)
```
## Usage with LiteLLM Proxy
### 1. Set Scaleway models in config.yaml
```yaml
model_list:
- model_name: scaleway-model
litellm_params:
model: scaleway/qwen3-235b-a22b-instruct-2507
api_key: "os.environ/SCW_SECRET_KEY" # ensure you have `SCW_SECRET_KEY` in your .env
```
### 2. Start proxy
```bash
litellm --config config.yaml
```
### 3. Query proxy
Assuming the proxy is running on [http://localhost:4000](http://localhost:4000):
```bash
curl http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \
-d '{
"model": "scaleway-model",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Write a short poem"
}
]
}'
```
`-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" ` is only required if you have set a LiteLLM master key
## Supported features
Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling.
docs/my-website/docs/providers/watsonx/rerank.md
+52
View File
@@ -0,0 +1,52 @@
# watsonx.ai Rerank
## Overview
| Property | Details |
|----------|--------------------------------------------------------------------------|
| Description | watsonx.ai rerank integration |
| Provider Route on LiteLLM | `watsonx/` |
| Supported Operations | `/ml/v1/text/rerank` |
| Link to Provider Doc | [IBM WatsonX.ai ↗](https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank) |
## Quick Start
### **LiteLLM SDK**
```python
import os
from litellm import rerank
os.environ["WATSONX_APIKEY"] = "YOUR_WATSONX_APIKEY"
os.environ["WATSONX_API_BASE"] = "YOUR_WATSONX_API_BASE"
os.environ["WATSONX_PROJECT_ID"] = "YOUR_WATSONX_PROJECT_ID"
query="Best programming language for beginners?"
documents=[
"Python is great for beginners due to simple syntax.",
"JavaScript runs in browsers and is versatile.",
"Rust has a steep learning curve but is very safe.",
]
response = rerank(
model="watsonx/cross-encoder/ms-marco-minilm-l-12-v2",
query=query,
documents=documents,
top_n=2,
return_documents=True,
)
print(response)
```
### **LiteLLM Proxy**
```yaml
model_list:
- model_name: cross-encoder/ms-marco-minilm-l-12-v2
litellm_params:
model: watsonx/cross-encoder/ms-marco-minilm-l-12-v2
api_key: os.environ/WATSONX_APIKEY
api_base: os.environ/WATSONX_API_BASE
project_id: os.environ/WATSONX_PROJECT_ID
```
docs/my-website/docs/proxy/access_groups.md
+122
View File
@@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Access Groups
Access Groups simplify how you define and manage resource access across your organization. Instead of configuring models, MCP servers, and agents separately on each key or team, you create one group that bundles the resources you want to grant, then attach that group to your keys or teams.
## Overview
**Access Groups** let you define a reusable set of allowed resources—models, MCP servers, and agents—in a single place. One group can grant access to all three resource types. Simply attach the group to a key or team, and they get access to everything defined in that group.
- **Unified resource control** – One group controls access to models, MCP servers, and agents together
- **Reusable** – Define once, attach to many keys or teams
- **Easy to maintain** – Update the group (add or remove resources) and all attached keys and teams automatically reflect the change
- **Clear visibility** – See exactly which resources each group grants and which keys/teams use it
<Image img={require('../../img/ui_access_groups.png')} />
### How It Works
**Key concept:** Define resources in a group → Attach group to key or team → Key/team gets access to all resources in the group
| Resource Type | What the group controls |
| --------------- | -------------------------------------------------------------------- |
| **Models** | Which LLM models keys/teams can use (e.g., `gpt-4`, `claude-3-opus`) |
| **MCP Servers** | Which MCP servers are available for tool calling |
| **Agents** | Which agents can be invoked |
## How to Create and Use Access Groups in the UI
### 1. Navigate to Access Groups
Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Access Groups** in the sidebar.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/d117fdb2-18c8-49e0-91e6-1f830d2d4b85/ascreenshot_f5822a0ddac64e3383124419d0c66298_text_export.jpeg)
### 2. Create an Access Group
Click **Create Access Group** and give your group a name.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/aefb900d-d106-4436-806c-3608ad19659f/ascreenshot_3f6fed1256604fe3b7038a0778ce3342_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/0951bb93-61bd-477e-beaf-f58810f8980b/ascreenshot_f0fb5d552fd74ff8a1080e82758fcdc2_text_export.jpeg)
### 3. Define Resources in the Group
Use the tabs to select which models, MCP servers, and agents this group grants access to:
- **Models tab** – Select the LLM models
- **MCP Servers tab** – Select MCP servers (for tool calling)
- **Agents tab** – Select agents
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/37398e8f-cd50-48c9-85e2-c77b2eeb994b/ascreenshot_440ec7906c8f4199b30ef91c903960b9_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/99d36543-8582-4bb7-a34d-3d5fe0fcf12f/ascreenshot_d9983240955c496892e1f7c38c074045_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/06fc5919-5c71-4fc3-999b-da7a4800af3f/ascreenshot_db93fdf742b249dc90a4b9d5991d6097_text_export.jpeg)
### 4. Attach the Access Group to a Key
When creating or editing a virtual key, expand **Optional Settings** and select your Access Group. The key will inherit access to all models, MCP servers, and agents defined in that group.
1. Go to **Virtual Keys** and click **+ Create New Key**
2. Expand **Optional Settings**
3. In the Access Group field, select the group you created
4. Save the key
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/cdfa76ab-bf38-4ca4-a97d-2cb50fafe50b/ascreenshot_046daecb57554c28ba553cf6c01f5450_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/84f08e9c-e9d0-42aa-8317-f385190b6d7d/ascreenshot_2d239716d30f431d9ad494baf7933d6a_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/41d7b7f9-ac58-4602-b887-c35c9b419dce/ascreenshot_8abd4fef48014dd1b88848411e6d7912_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/e37b01c0-f2d7-4133-8b2f-ccc51f6769e1/ascreenshot_f495df428ad54cac9ec43b46c3dfc1b1_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/3fe33cad-6b64-46c3-a66e-6e6e073c3d7a/ascreenshot_f2dcc79ae8af47dd86ade2f85165d3c1_text_export.jpeg)
### 5. Attach the Access Group to a Team
You can also attach an Access Group to a team when creating or editing the team. All keys associated with that team will then have access to the resources defined in the group.
## Use Cases
### Team-based Access
Create groups like "Engineering", "Data Science", or "Product" with the models, MCP servers, and agents each team needs. Attach the group to the team—no need to configure each resource on every key.
### Environment Separation
- **Production group** – Production models, approved MCP servers, and production agents
- **Development group** – Cost-efficient models, experimental MCP tools, and dev agents
Attach the appropriate group to keys or teams based on environment.
### Simplified Onboarding
New developers get a key with an Access Group instead of manually configuring models, MCP servers, and agents. Add them to the right team or give them a key with the correct group.
### Centralized Updates
When you add a new model or MCP server to a group, every key and team attached to that group automatically gains access. Remove a resource from the group and it’s revoked everywhere at once.
## Access Group vs. Model Access Groups
LiteLLM has two related concepts:
| Feature | **Access Groups** (this page) | **Model Access Groups** |
| ---------- | ----------------------------------------------------------------------- | ------------------------------------------------------- |
| Definition | Define in the UI; one group can include models, MCP servers, and agents | Defined in config or via API; groups are model-centric |
| Scope | Models + MCP servers + agents | Models only |
| Attach to | Keys, teams | Keys, teams |
| Use when | You want unified control over models, MCP, and agents from the UI | You need config-based or API-based model access control |
For config-based model access with `access_groups` in `model_info`, see [Model Access Groups](./model_access_groups.md).
## Related Documentation
- [Virtual Keys](./virtual_keys.md) – Creating and managing API keys
- [Role-based Access Controls](./access_control.md) – Organizations, teams, and user roles
- [Model Access Groups](./model_access_groups.md) – Config-based model access groups
- [MCP Control](../mcp_control.md) – MCP server setup and access control
docs/my-website/docs/proxy/auto_routing.md
+186
View File
@@ -219,3 +219,189 @@ curl -X POST http://localhost:4000/v1/chat/completions \
3. If a route's similarity score exceeds the threshold, the request is routed to that model
4. If no route matches, the request goes to the default model
---
## Complexity Router
The Complexity Router provides an alternative to semantic routing that uses **rule-based scoring** to classify requests by complexity and route them to appropriate models — with **zero external API calls** and **sub-millisecond latency**.
### When to Use
| Feature | Semantic Auto Router | Complexity Router |
|---------|---------------------|-------------------|
| Classification | Embedding-based matching | Rule-based scoring |
| Latency | ~100-500ms (embedding API) | &lt;1ms |
| API Calls | Requires embedding model | None |
| Training | Requires utterance examples | Works out of the box |
| Best For | Intent-based routing | Cost optimization |
Use **Complexity Router** when you want to:
- Route simple queries to cheaper/faster models (e.g., gpt-4o-mini)
- Route complex queries to more capable models (e.g., claude-sonnet-4)
- Minimize latency overhead from routing decisions
- Avoid additional API costs for embeddings
### LiteLLM Python SDK
```python
from litellm import Router
router = Router(
model_list=[
# Target models for each tier
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "gpt-4o-mini"},
},
{
"model_name": "gpt-4o",
"litellm_params": {"model": "gpt-4o"},
},
{
"model_name": "claude-sonnet",
"litellm_params": {"model": "claude-sonnet-4-20250514"},
},
{
"model_name": "o1-preview",
"litellm_params": {"model": "o1-preview"},
},
# Complexity router configuration
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {
"SIMPLE": "gpt-4o-mini",
"MEDIUM": "gpt-4o",
"COMPLEX": "claude-sonnet",
"REASONING": "o1-preview",
},
},
"complexity_router_default_model": "gpt-4o",
},
},
],
)
```
#### Usage
```python
# Simple query → routes to gpt-4o-mini
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
# Complex technical query → routes to claude-sonnet or higher
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "Design a distributed microservice architecture with Kubernetes orchestration"}],
)
# Reasoning request → routes to o1-preview
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "Think step by step and reason through this problem carefully..."}],
)
```
### LiteLLM Proxy Server
Add the complexity router to your `config.yaml`:
```yaml
model_list:
# Target models
- model_name: gpt-4o-mini
litellm_params:
model: gpt-4o-mini
- model_name: gpt-4o
litellm_params:
model: gpt-4o
- model_name: claude-sonnet
litellm_params:
model: claude-sonnet-4-20250514
- model_name: o1-preview
litellm_params:
model: o1-preview
# Complexity router
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet
REASONING: o1-preview
complexity_router_default_model: gpt-4o
```
### Configuration Options
#### Tier Boundaries
Customize the score thresholds for each tier:
```yaml
complexity_router_config:
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet
REASONING: o1-preview
tier_boundaries:
simple_medium: 0.15 # Below 0.15 → SIMPLE
medium_complex: 0.35 # 0.15-0.35 → MEDIUM
complex_reasoning: 0.60 # 0.35-0.60 → COMPLEX, above → REASONING
```
#### Token Thresholds
Adjust when prompts are considered "short" or "long":
```yaml
complexity_router_config:
token_thresholds:
simple: 15 # Prompts under 15 tokens are penalized (simple indicator)
complex: 400 # Prompts over 400 tokens get complexity boost
```
#### Dimension Weights
Customize how much each signal contributes to the complexity score:
```yaml
complexity_router_config:
dimension_weights:
tokenCount: 0.10 # Prompt length
codePresence: 0.30 # Code-related keywords
reasoningMarkers: 0.25 # "step by step", "think through", etc.
technicalTerms: 0.25 # Domain-specific complexity
simpleIndicators: 0.05 # "what is", "define", greetings
multiStepPatterns: 0.03 # "first...then", numbered steps
questionComplexity: 0.02 # Multiple questions
```
### How Complexity Routing Works
The router scores each request across 7 dimensions:
| Dimension | What It Detects | Effect |
|-----------|-----------------|--------|
| Token Count | Short (&lt;15) or long (&gt;400) prompts | Short = simple, long = complex |
| Code Presence | "function", "class", "api", "database", etc. | Increases complexity |
| Reasoning Markers | "step by step", "think through", "analyze" | Triggers REASONING tier |
| Technical Terms | "architecture", "distributed", "encryption" | Increases complexity |
| Simple Indicators | "what is", "define", "hello" | Decreases complexity |
| Multi-Step Patterns | "first...then", "1. 2. 3." | Increases complexity |
| Question Complexity | Multiple question marks | Increases complexity |
**Special behavior:** If 2+ reasoning markers are detected in the user message, the request automatically routes to the REASONING tier regardless of the weighted score.
docs/my-website/docs/proxy/config_settings.md
+21 -4
View File
@@ -358,7 +358,8 @@ router_settings:
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `deployment_affinity`, `forward_client_headers_by_model_group` |
| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
@@ -450,6 +451,7 @@ router_settings:
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
| BEDROCK_MIN_THINKING_BUDGET_TOKENS | Minimum thinking budget in tokens for Bedrock reasoning models. Bedrock returns a 400 error if budget_tokens is below this value. Requests with lower values are clamped to this minimum. Default is 1024
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
@@ -483,6 +485,7 @@ router_settings:
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
| COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3
| DATABASE_HOST | Hostname for the database server
| DATABASE_NAME | Name of the database
| DATABASE_PASSWORD | Password for the database user
@@ -492,6 +495,7 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
| DATABRICKS_API_KEY | API key (Personal Access Token) for Databricks API authentication
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
@@ -520,6 +524,7 @@ router_settings:
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
| DEFAULT_ACCESS_GROUP_CACHE_TTL | Time-to-live in seconds for cached access group information. Default is 600 (10 minutes)
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
@@ -538,7 +543,7 @@ router_settings:
| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5
| DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL | Default time-to-live in seconds for management objects (User, Team, Key, Organization) in memory cache. Default is 60 seconds.
| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16
| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 64
| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100
| DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER | Default maximum recursion depth for sensitive data masker. Default is 10
| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2
@@ -548,6 +553,7 @@ router_settings:
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
@@ -555,7 +561,7 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy when `NUM_WORKERS` is not set. Default is 1. **We strongly recommend setting NUM_WORKERS to the number of vCPUs available** (e.g. `NUM_WORKERS=8` or `--num_workers 8`).
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
@@ -600,7 +606,6 @@ router_settings:
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for budget alert deduplication in seconds. Default is 86400 (24 hours)
| ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
| ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@@ -745,9 +750,12 @@ router_settings:
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
@@ -760,11 +768,15 @@ router_settings:
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_MODEL_COST_MAP_URL | URL for fetching model cost map data. Default is https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
@@ -772,6 +784,10 @@ router_settings:
| LITELLM_METER_NAME | Name for OTEL Meter
| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling).
| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
@@ -791,6 +807,7 @@ router_settings:
| LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
| LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
| LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50%
| MAX_COMPETITOR_NAMES | Maximum number of competitor names allowed in policy template enrichment. Default is 100
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200
| MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0
docs/my-website/docs/proxy/customers.md
+248 -15
View File
@@ -2,29 +2,98 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Customers / End-User Budgets
# Customers / End-Users
Track spend, set budgets for your customers.
Track spend, set budgets and permissions for your customers.
## Tracking Customer Spend
## Tracking Customer Spend + Permissions
### 1. Make LLM API call w/ Customer ID
Make a /chat/completions call, pass 'user' - First call Works
LiteLLM checks for a customer/end-user ID in the following order (first match wins):
```bash showLineNumbers title="Make request with customer ID"
| Priority | Method | Where | Notes |
|----------|--------|-------|-------|
| 1 | `x-litellm-customer-id` header | Request headers | Standard header, always checked |
| 2 | `x-litellm-end-user-id` header | Request headers | Standard header, always checked |
| 3 | Custom header via `user_header_mappings` | Request headers | Configured in `general_settings` |
| 4 | Custom header via `user_header_name` | Request headers | Deprecated — use `user_header_mappings` |
| 5 | `user` field | Request body | Standard OpenAI field |
| 6 | `litellm_metadata.user` field | Request body | Anthropic-style metadata |
| 7 | `metadata.user_id` field | Request body | Generic metadata pattern |
| 8 | `safety_identifier` field | Request body | Responses API |
**Option 1: Standard headers** (recommended — no request body modification needed)
```bash showLineNumbers title="Make request with customer ID in header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data ' {
--header 'Authorization: Bearer sk-1234' \
--header 'x-litellm-end-user-id: ishaan3' \
--data '{
"model": "azure-gpt-3.5",
"user": "ishaan3", # 👈 CUSTOMER ID
"messages": [
{
"role": "user",
"content": "what time is it"
}
]
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
Both `x-litellm-customer-id` and `x-litellm-end-user-id` are supported and always checked without any configuration.
**Option 2: `user` field in request body** (OpenAI-compatible)
```bash showLineNumbers title="Make request with customer ID in body"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-gpt-3.5",
"user": "ishaan3",
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
**Option 3: Custom header via `user_header_mappings`** (configurable)
```yaml showLineNumbers title="config.yaml"
general_settings:
user_header_mappings:
- header_name: "x-my-app-user-id"
litellm_user_role: "customer"
```
```bash showLineNumbers title="Make request with custom header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'x-my-app-user-id: ishaan3' \
--data '{
"model": "azure-gpt-3.5",
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
**Option 4: `litellm_metadata.user`** (Anthropic-style)
```bash showLineNumbers title="Make request with litellm_metadata.user"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "what time is it"}],
"litellm_metadata": {"user": "ishaan3"}
}'
```
**Option 5: `metadata.user_id`**
```bash showLineNumbers title="Make request with metadata.user_id"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-gpt-3.5",
"messages": [{"role": "user", "content": "what time is it"}],
"metadata": {"user_id": "ishaan3"}
}'
```
@@ -123,7 +192,171 @@ Expected Response
</Tabs>
## Setting Customer Budgets
## Setting Customer Object Permissions
Control which resources (MCP servers, vector stores, agents) a customer can access.
### What are Object Permissions?
Object permissions allow you to restrict customer access to specific:
- **MCP Servers**: Limit which MCP servers the customer can call
- **MCP Access Groups**: Assign customers to predefined groups of MCP servers
- **MCP Tool Permissions**: Granular control over which tools within an MCP server the customer can use
- **Vector Stores**: Control which vector stores the customer can query
- **Agents**: Restrict which agents the customer can interact with
- **Agent Access Groups**: Assign customers to predefined groups of agents
### Creating a Customer with Object Permissions
```bash showLineNumbers title="Create customer with object permissions"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "user_1",
"object_permission": {
"mcp_servers": ["server_1", "server_2"],
"mcp_access_groups": ["public_group"],
"mcp_tool_permissions": {
"server_1": ["tool_a", "tool_b"]
},
"vector_stores": ["vector_store_1"],
"agents": ["agent_1"],
"agent_access_groups": ["basic_agents"]
}
}'
```
**Parameters:**
- `mcp_servers` (Optional[List[str]]): List of allowed MCP server IDs
- `mcp_access_groups` (Optional[List[str]]): List of MCP access group names
- `mcp_tool_permissions` (Optional[Dict[str, List[str]]]): Map of server ID to allowed tool names
- `vector_stores` (Optional[List[str]]): List of allowed vector store IDs
- `agents` (Optional[List[str]]): List of allowed agent IDs
- `agent_access_groups` (Optional[List[str]]): List of agent access group names
**Note:** If `object_permission` is `null` or `{}`, the customer has no object-level restrictions.
### Updating Customer Object Permissions
You can update object permissions for existing customers:
```bash showLineNumbers title="Update customer object permissions"
curl -L -X POST 'http://localhost:4000/customer/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "user_1",
"object_permission": {
"mcp_servers": ["server_3"],
"vector_stores": ["vector_store_2", "vector_store_3"]
}
}'
```
### Viewing Customer Object Permissions
When you query customer info, object permissions are included in the response:
```bash showLineNumbers title="Get customer info with object permissions"
curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=user_1' \
-H 'Authorization: Bearer sk-1234'
```
**Response:**
```json showLineNumbers title="Response with object permissions"
{
"user_id": "user_1",
"blocked": false,
"alias": "John Doe",
"spend": 0.0,
"object_permission": {
"object_permission_id": "perm_abc123",
"mcp_servers": ["server_1", "server_2"],
"mcp_access_groups": ["public_group"],
"mcp_tool_permissions": {
"server_1": ["tool_a", "tool_b"]
},
"vector_stores": ["vector_store_1"],
"agents": ["agent_1"],
"agent_access_groups": ["basic_agents"]
},
"litellm_budget_table": null
}
```
### Use Cases
**1. Tiered Access Control**
Create different permission tiers for your customers:
```bash showLineNumbers title="Free tier customer"
# Free tier - limited access
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "free_user",
"budget_id": "free_tier",
"object_permission": {
"mcp_access_groups": ["public_group"],
"agent_access_groups": ["basic_agents"]
}
}'
```
```bash showLineNumbers title="Premium tier customer"
# Premium tier - full access
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "premium_user",
"budget_id": "premium_tier",
"object_permission": {
"mcp_servers": ["server_1", "server_2", "server_3"],
"vector_stores": ["vector_store_1", "vector_store_2"],
"agents": ["agent_1", "agent_2", "agent_3"]
}
}'
```
**2. Department-Specific Access**
Restrict customers to resources relevant to their department:
```bash showLineNumbers title="Sales team customer"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "sales_user",
"object_permission": {
"mcp_servers": ["crm_server", "email_server"],
"agents": ["sales_assistant"],
"vector_stores": ["sales_knowledge_base"]
}
}'
```
**3. Tool-Level Restrictions**
Grant access to specific tools within an MCP server:
```bash showLineNumbers title="Limited tool access"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "restricted_user",
"object_permission": {
"mcp_servers": ["database_server"],
"mcp_tool_permissions": {
"database_server": ["read_only_query", "get_table_schema"]
}
}
}'
```
## Setting Customer Budgets
Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy
docs/my-website/docs/proxy/guardrails/policy_templates.md
+296
View File
@@ -0,0 +1,296 @@
# Policy Templates
Policy templates provide pre-configured guardrail policies that you can use as a starting point for your organization. Instead of manually creating policies and guardrails, you can select a template that matches your use case and deploy it with one click.
## Using Policy Templates
### In the UI
1. Navigate to **Policies → Templates** tab in the LiteLLM Admin UI
2. Browse available templates (e.g., "PII Protection", "Cost Control", "HR Compliance")
3. Click **"Use Template"** on any template
4. Review the guardrails that will be created:
- Existing guardrails are marked with a green checkmark
- New guardrails can be selected/deselected
5. Click **"Create X Guardrails & Use Template"**
6. Review and customize the pre-filled policy form
7. Click **"Create Policy"** to save
### Workflow
```
Select Template → Review Guardrails → Create Selected → Edit Policy → Save
```
The system automatically:
- ✅ Detects which guardrails already exist
- ✅ Creates only the missing guardrails you select
- ✅ Pre-fills the policy form with template data
- ✅ Lets you customize before saving
## Available Templates
Templates are fetched from [GitHub](https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json) with automatic fallback to local backup.
### Current Templates
#### 1. Advanced PII Protection (Australia)
- **Complexity:** High
- **Use Case:** Comprehensive PII detection for Australian organizations
- **Guardrails:**
- Australian tax identifiers (TFN, ABN, Medicare)
- Australian passports
- International PII (SSN, passports, national IDs)
- Contact information (email, phone, address)
- Financial data (credit cards, IBAN)
- API credentials (AWS, GitHub, Slack) - **BLOCKS** requests
- Network infrastructure (IP addresses)
- Protected class information (gender, race, religion, disability, etc.)
#### 2. Baseline PII Protection
- **Complexity:** Low
- **Use Case:** Basic protection for internal tools and testing
- **Guardrails:**
- Australian tax identifiers
- API credentials
- Financial data
## Creating Your Own Policy Templates
You can contribute policy templates for the entire LiteLLM community to use.
### Template Structure
Templates are defined in JSON format with the following structure:
```json
{
"id": "unique-template-id",
"title": "Display Title",
"description": "Detailed description of what this template protects",
"icon": "ShieldCheckIcon",
"iconColor": "text-purple-500",
"iconBg": "bg-purple-50",
"guardrails": [
"guardrail-name-1",
"guardrail-name-2"
],
"complexity": "Low|Medium|High",
"guardrailDefinitions": [
{
"guardrail_name": "example-guardrail",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "email",
"action": "MASK"
}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "What this guardrail does"
}
}
],
"templateData": {
"policy_name": "policy-name",
"description": "Policy description",
"guardrails_add": ["guardrail-name-1", "guardrail-name-2"],
"guardrails_remove": []
}
}
```
### Field Descriptions
#### Display Fields
- **id**: Unique identifier (lowercase with hyphens)
- **title**: User-facing name shown in UI
- **description**: Detailed explanation of what the template protects
- **icon**: Icon name (must be available in UI icon map)
- **iconColor**: Tailwind CSS text color class
- **iconBg**: Tailwind CSS background color class
- **guardrails**: Array of guardrail names (for display only)
- **complexity**: Badge showing difficulty ("Low", "Medium", or "High")
#### Guardrail Definitions
- **guardrailDefinitions**: Array of complete guardrail configurations
- Each must be a valid guardrail object that can be sent to `/guardrails` POST endpoint
- If a guardrail already exists, it will be skipped
- Can be empty `[]` if template uses only existing guardrails
#### Policy Configuration
- **templateData**: Object that pre-fills the policy form
- **policy_name**: Suggested name (user can edit)
- **description**: Policy description
- **guardrails_add**: Array of guardrail names to include
- **guardrails_remove**: Array to remove (usually `[]` for templates)
- **inherit**: (Optional) Parent policy name for inheritance
### Example Template
Here's a complete example for a HIPAA compliance template:
```json
{
"id": "hipaa-compliance",
"title": "HIPAA Compliance Policy",
"description": "Healthcare compliance policy that masks PHI and enforces HIPAA regulations for healthcare applications.",
"icon": "ShieldCheckIcon",
"iconColor": "text-red-500",
"iconBg": "bg-red-50",
"guardrails": [
"phi-detector",
"medical-record-blocker",
"patient-id-masker"
],
"complexity": "High",
"guardrailDefinitions": [
{
"guardrail_name": "phi-detector",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "us_ssn",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "email",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "us_phone",
"action": "MASK"
}
],
"pattern_redaction_format": "[PHI_REDACTED]"
},
"guardrail_info": {
"description": "Detects and masks Protected Health Information (PHI)"
}
}
],
"templateData": {
"policy_name": "hipaa-compliance-policy",
"description": "HIPAA compliance policy for healthcare applications",
"guardrails_add": [
"phi-detector",
"medical-record-blocker",
"patient-id-masker"
],
"guardrails_remove": []
}
}
```
## Contributing Templates
To contribute a policy template for everyone to use:
### Step 1: Create Your Template JSON
1. Create a JSON file following the structure above
2. Test it locally by adding it to your local `policy_templates.json`
3. Verify all guardrails work correctly
4. Ensure descriptions are clear and helpful
### Step 2: Submit a Pull Request
1. Fork the [LiteLLM repository](https://github.com/BerriAI/litellm)
2. Add your template to `policy_templates.json` at the root
3. Add your template to `litellm/policy_templates_backup.json` (keep both in sync)
4. Create a pull request with:
- Clear description of what the template protects
- Use case examples
- Any relevant compliance frameworks (HIPAA, GDPR, SOC 2, etc.)
### Guidelines
**DO:**
- ✅ Use clear, descriptive names
- ✅ Include comprehensive descriptions
- ✅ Test all guardrails thoroughly
- ✅ Document pattern sources (e.g., "Based on NIST guidelines")
- ✅ Group related guardrails logically
- ✅ Consider different complexity levels
**DON'T:**
- ❌ Include credentials or secrets
- ❌ Use overly broad patterns that may have false positives
- ❌ Duplicate existing templates
- ❌ Use custom code without thorough testing
## Using Templates Offline
For air-gapped or offline deployments, set the environment variable:
```bash
export LITELLM_LOCAL_POLICY_TEMPLATES=true
```
This forces the system to use the local backup (`litellm/policy_templates_backup.json`) instead of fetching from GitHub.
## Template Sources
- **GitHub (default):** https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json
- **Local backup:** `litellm/policy_templates_backup.json`
Templates are automatically fetched from GitHub on each request, with fallback to local backup on any failure.
## Available Pattern Types
When creating guardrails for templates, you can use these prebuilt patterns:
### Identity Documents
- `passport_australia`, `passport_us`, `passport_uk`, `passport_germany`, etc.
- `us_ssn`, `us_ssn_no_dash`
- `au_tfn`, `au_abn`, `au_medicare`
- `nl_bsn_contextual`
- `br_cpf`, `br_rg`, `br_cnpj`
### Financial
- `visa`, `mastercard`, `amex`, `discover`, `credit_card`
- `iban`
### Contact Information
- `email`
- `us_phone`, `br_phone_landline`, `br_phone_mobile`
- `street_address`
- `br_cep` (Brazilian postal code)
### Credentials
- `aws_access_key`, `aws_secret_key`
- `github_token`
- `slack_token`
- `generic_api_key`
### Network
- `ipv4`, `ipv6`
### Protected Class
- `gender_sexual_orientation`
- `race_ethnicity_national_origin`
- `religion`
- `age_discrimination`
- `disability`
- `marital_family_status`
- `military_status`
- `public_assistance`
See the [full patterns list](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json) for all available patterns.
## Related Docs
- [Guardrail Policies](./guardrail_policies)
- [Policy Tags](./policy_tags)
- [Content Filter Patterns](../hooks/content_filter)
- [Custom Code Guardrails](../hooks/custom_code)
docs/my-website/docs/proxy/logging.md
+1
View File
@@ -1338,6 +1338,7 @@ litellm_settings:
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
s3_use_virtual_hosted_style: false # [OPTIONAL] use virtual-hosted-style URLs (bucket.endpoint/key) instead of path-style (endpoint/bucket/key). Useful for S3-compatible services like MinIO
s3_strip_base64_files: false # [OPTIONAL] remove base64 files before storing in s3
```
docs/my-website/docs/proxy/pass_through.md
+110 -1
View File
@@ -58,6 +58,17 @@ Configure the required authentication and pricing:
- The Bria API requires an `api_token` header
- Enter your Bria API key as the value for the `api_token` header
**Default Query Parameters (Optional):**
- Add query parameters that will be automatically sent with every request
- Perfect for API versioning, format specifications, or default configurations
- Clients can override these parameters by providing their own values
- Example: `version=v1`, `format=json`, `timeout=30`
<Image
img={require('../../img/passthrough_query_default.png')}
style={{width: '60%', display: 'block', margin: '2rem auto'}}
/>
**Pricing Configuration:**
- Set a cost per request (e.g., $12.00 in this example)
- This enables cost tracking and billing for your users
@@ -112,6 +123,9 @@ general_settings:
content-type: application/json
accept: application/json
forward_headers: true # Forward all incoming headers
default_query_params: # Optional: Default query parameters
version: "v1" # Always send version=v1
format: "json" # Default format (can be overridden)
```
### Start and Test
@@ -166,6 +180,9 @@ general_settings:
auth: boolean # Enable LiteLLM authentication (Enterprise)
forward_headers: boolean # Forward all incoming headers
include_subpath: boolean # If true, forwards requests to sub-paths (default: false)
methods: list[string] # Optional: HTTP methods (e.g., ["GET", "POST"]). If not specified, all methods are supported.
default_query_params: # Optional: Default query parameters sent with every request
<param-name>: string # Key-value pairs (e.g., version: "v1", format: "json")
headers: # Custom headers to add
Authorization: string # Auth header for target API
content-type: string # Request content type
@@ -177,11 +194,17 @@ general_settings:
### Header Options
- **Authorization**: Authentication for the target API
- **content-type**: Request body format specification
- **content-type**: Request body format specification
- **accept**: Expected response format
- **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration
- **Custom headers**: Any additional key-value pairs
### Default Query Parameters
- **Parameter precedence**: Client params > URL params > default params
- **Use cases**: API versioning, authentication tokens, format control, feature flags
- **Override capability**: Clients can override any default parameter
- **Examples**: `version: "v1"`, `format: "json"`, `timeout: "30"`
### Sub-path Routing
By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`:
@@ -201,6 +224,92 @@ general_settings:
---
### Default Query Parameters
Pass-through endpoints support default query parameters that are automatically added to every request. This is useful for API versioning, format specifications, authentication tokens, or any default configuration.
#### How It Works
**Parameter Precedence (highest to lowest priority):**
1. **Client-provided parameters** (in the request URL)
2. **URL parameters** (from the target URL)
3. **Default parameters** (from configuration)
#### Example Configuration
```yaml
general_settings:
pass_through_endpoints:
- path: "/api/v1"
target: "https://external-api.com/service?timeout=60" # URL has timeout=60
default_query_params:
version: "v1" # Always add version=v1
format: "json" # Default format=json (can be overridden)
auth_level: "basic" # Always add auth_level=basic
```
#### Request Examples
**Client Request:** `GET /api/v1/users`
**Actual Backend Call:** `https://external-api.com/service?version=v1&format=json&auth_level=basic&timeout=60`
**Client Request:** `GET /api/v1/users?format=xml&custom=value`
**Actual Backend Call:** `https://external-api.com/service?version=v1&auth_level=basic&timeout=60&format=xml&custom=value`
- Client `format=xml` overrides default `format=json`
- Default `version=v1` and `auth_level=basic` are preserved
- URL `timeout=60` is preserved
- Client `custom=value` is added
#### Use Cases
- **API Versioning**: Always send `version=v2` to maintain compatibility
- **Authentication**: Add authentication tokens like `api_key=default_key`
- **Format Control**: Default to `format=json` but allow client override
- **Rate Limiting**: Set `rate_limit=standard` as default
- **Feature Flags**: Enable `experimental=false` by default
---
You can configure different target URLs for the same path using different HTTP methods. This is useful when different backends handle different operations:
<Image
img={require('../../img/passthrough_method_setup.png')}
style={{width: '60%', display: 'block', margin: '2rem auto'}}
/>
```yaml
general_settings:
pass_through_endpoints:
# GET requests to /azure/kb go to read API
- path: "/azure/kb"
target: "https://read-api.example.com/knowledge-base"
methods: ["GET"]
headers:
Authorization: "bearer os.environ/READ_API_KEY"
# POST requests to /azure/kb go to write API
- path: "/azure/kb"
target: "https://write-api.example.com/knowledge-base"
methods: ["POST"]
headers:
Authorization: "bearer os.environ/WRITE_API_KEY"
# PUT requests to /azure/kb go to update API
- path: "/azure/kb"
target: "https://update-api.example.com/knowledge-base"
methods: ["PUT"]
headers:
Authorization: "bearer os.environ/UPDATE_API_KEY"
```
**Key Points:**
- If `methods` is not specified, the endpoint supports all HTTP methods (GET, POST, PUT, DELETE, PATCH)
- Multiple endpoints can share the same path as long as they have different methods
- You can specify multiple methods for a single endpoint: `methods: ["GET", "POST"]`
- This allows you to route to different backends based on the operation type
---
## Advanced: Custom Adapters
For complex integrations (like Anthropic/Bedrock clients), you can create custom adapters that translate between different API schemas.
docs/my-website/docs/proxy/project_management.md
+318
View File
@@ -0,0 +1,318 @@
# [Beta] Project Management
Projects in LiteLLM sit between teams and keys in the organizational hierarchy, enabling fine-grained access control and budget management for specific use cases or applications.
```mermaid
graph TD
A[Organization] --> B[Team 1]
A --> C[Team 2]
B --> D[Project A]
B --> E[Project B]
C --> F[Project C]
D --> G[API Key 1]
D --> H[API Key 2]
E --> I[API Key 3]
F --> J[API Key 4]
style A fill:#e1f5ff
style B fill:#fff4e6
style C fill:#fff4e6
style D fill:#f3e5f5
style E fill:#f3e5f5
style F fill:#f3e5f5
style G fill:#e8f5e9
style H fill:#e8f5e9
style I fill:#e8f5e9
style J fill:#e8f5e9
```
**Hierarchy**: `Organizations > Teams > Projects > Keys`
## Quick Start
This walkthrough shows how to create a project, generate an API key, make requests, and view project-level spend tracking in the UI.
### Step 1: Create a Project
```bash showLineNumbers
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "flight-search-assistant",
"team_id": "ad898803-c8a3-4f4a-976a-a3c372cffa45",
"models": ["gpt-4", "gpt-3.5-turbo"],
"max_budget": 100,
"metadata": {
"use_case_id": "SNOW-12345",
"responsible_ai_id": "RAI-67890"
}
}' | jq
```
**Response:**
```json
{
"project_id": "e402a141-725a-4437-bff5-d47459189716",
"project_alias": "flight-search-assistant",
"team_id": "ad898803-c8a3-4f4a-976a-a3c372cffa45",
"models": ["gpt-4", "gpt-3.5-turbo"],
"max_budget": 100,
...
}
```
### Step 2: Generate API Key for Project
```bash showLineNumbers
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data-raw '{
"models": ["gpt-3.5-turbo", "gpt-4"],
"metadata": {"user": "ishaan@berri.ai"},
"project_id": "e402a141-725a-4437-bff5-d47459189716"
}' | jq
```
**Response:**
```json
{
"key": "sk-W8VbscpfuyvHm5TkxRYiXA",
"key_name": "sk-...YiXA",
"project_id": "e402a141-725a-4437-bff5-d47459189716",
...
}
```
### Step 3: Use API Key in Chat Completions
```bash showLineNumbers
curl http://localhost:4000/v1/chat/completions \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-W8VbscpfuyvHm5TkxRYiXA' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "What is litellm?"}]
}' | jq
```
### Step 4: View Project Spend in UI
Navigate to the **Logs** page in the LiteLLM Admin UI. You'll see the `user_api_key_project_id` tracked in the request metadata:
![Project Spend Tracking](/img/project_spend.png)
As shown above, the spend logs metadata includes:
- `"user_api_key_project_id": "e402a141-725a-4437-bff5-d47459189716"` - Links the request to your project
- All costs and token usage are automatically attributed to the project
- You can query and filter logs by project ID for detailed reporting
## API Endpoints
### POST /project/new
Create a new project.
**Who can call**: Admins or Team Admins
**Parameters**:
- `project_alias` (string, optional): Human-readable name for the project
- `team_id` (string, required): The team this project belongs to
- `models` (array, optional): List of models the project can access
- `max_budget` (float, optional): Maximum spend budget for the project
- `tpm_limit` (int, optional): Tokens per minute limit
- `rpm_limit` (int, optional): Requests per minute limit
- `budget_duration` (string, optional): Budget reset period (e.g., "30d", "1mo")
- `metadata` (object, optional): Custom metadata for the project
- `blocked` (boolean, optional): Block all API calls for this project
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"models": ["claude-3-sonnet"],
"max_budget": 200,
"tpm_limit": 100000,
"metadata": {
"use_case_id": "SNOW-12346",
"cost_center": "travel-products"
}
}'
```
**Response**:
```json
{
"project_id": "project-def",
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"models": ["claude-3-sonnet"],
"spend": 0.0,
"budget_id": "budget-xyz",
"metadata": {
"use_case_id": "SNOW-12346",
"cost_center": "travel-products"
},
"created_at": "2025-01-15T10:00:00Z",
"updated_at": "2025-01-15T10:00:00Z"
}
```
### POST /project/update
Update an existing project.
**Who can call**: Admins or Team Admins
**Parameters**:
- `project_id` (string, required): The project to update
- `project_alias` (string, optional): Updated project name
- `team_id` (string, optional): Move project to different team
- `models` (array, optional): Updated list of allowed models
- `max_budget` (float, optional): Updated budget
- `tpm_limit` (int, optional): Updated TPM limit
- `rpm_limit` (int, optional): Updated RPM limit
- `metadata` (object, optional): Updated metadata
- `blocked` (boolean, optional): Updated blocked status
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_id": "project-abc",
"max_budget": 200,
"tpm_limit": 200000,
"metadata": {
"status": "production"
}
}'
```
### GET /project/info
Get information about a specific project.
**Parameters**:
- `project_id` (string, required): Query parameter
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/info?project_id=project-abc' \
--header 'Authorization: Bearer sk-1234'
```
**Response**:
```json
{
"project_id": "project-abc",
"project_alias": "flight-search-assistant",
"team_id": "team-123",
"models": ["gpt-4", "gpt-3.5-turbo"],
"spend": 45.67,
"model_spend": {
"gpt-4": 42.30,
"gpt-3.5-turbo": 3.37
},
"litellm_budget_table": {
"budget_id": "budget-xyz",
"max_budget": 100.0,
"tpm_limit": 100000,
"rpm_limit": 100
},
"metadata": {
"use_case_id": "SNOW-12345"
}
}
```
### GET /project/list
List all projects the user has access to.
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/list' \
--header 'Authorization: Bearer sk-1234'
```
**Response**:
```json
[
{
"project_id": "project-abc",
"project_alias": "flight-search-assistant",
"team_id": "team-123",
"spend": 45.67
},
{
"project_id": "project-def",
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"spend": 23.45
}
]
```
### DELETE /project/delete
Delete one or more projects.
**Who can call**: Admins only
**Parameters**:
- `project_ids` (array, required): List of project IDs to delete
**Example**:
```bash
curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_ids": ["project-abc", "project-def"]
}'
```
**Note**: Projects with associated API keys cannot be deleted. Delete or reassign the keys first.
## Model-Specific Quotas
You can set different quotas for different models within a project:
```bash
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "multi-model-project",
"team_id": "team-123",
"models": ["gpt-4", "gpt-3.5-turbo", "claude-3-sonnet"],
"max_budget": 500,
"metadata": {
"model_tpm_limit": {
"gpt-4": 50000,
"gpt-3.5-turbo": 200000,
"claude-3-sonnet": 100000
},
"model_rpm_limit": {
"gpt-4": 50,
"gpt-3.5-turbo": 500,
"claude-3-sonnet": 100
}
}
}'
```
docs/my-website/docs/proxy/prompt_management.md
+54 -1
View File
@@ -11,6 +11,7 @@ Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini fin
| Native LiteLLM GitOps (.prompt files) | [Get Started](native_litellm_prompt) |
| Langfuse | [Get Started](https://langfuse.com/docs/prompts/get-started) |
| Humanloop | [Get Started](../observability/humanloop) |
| Generic Prompt Management API | [Get Started](../adding_provider/generic_prompt_management_api) |
## Onboarding Prompts via config.yaml
@@ -34,7 +35,7 @@ prompts:
- prompt_id: "my_prompt_id"
litellm_params:
prompt_id: "my_prompt_id"
prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, custom
prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, generic_prompt_management, custom
# integration-specific parameters below
```
@@ -46,6 +47,7 @@ The `prompt_integration` field determines where and how prompts are loaded:
- **`langfuse`**: Fetch prompts from Langfuse prompt management
- **`bitbucket`**: Load from BitBucket repository `.prompt` files (team-based access control)
- **`gitlab`**: Load from GitLab repository `.prompt` files (team-based access control)
- **`generic_prompt_management`**: Integrate any prompt management system via a simple API endpoint (no PR required)
- **`custom`**: Use your own custom prompt management implementation
Each integration has its own configuration parameters and access control mechanisms.
@@ -207,6 +209,57 @@ System: You are a helpful assistant.
User: {{user_message}}
```
</TabItem>
<TabItem value="generic" label="Generic Prompt Management">
```yaml
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
provider_specific_query_params:
project_name: litellm
slug: hello-world-prompt-2bac
api_base: http://localhost:8080
api_key: os.environ/GENERIC_PROMPT_API_KEY
ignore_prompt_manager_model: true # optional
ignore_prompt_manager_optional_params: true # optional
```
**What you need to implement:**
A GET endpoint at `/beta/litellm_prompt_management` that returns:
```json
{
"prompt_id": "simple_prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Help me with {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
**Benefits:**
- No PR required - integrate any prompt management system
- Full control over your prompt storage and versioning
- Support for variable substitution with `{variable}` syntax
- Custom query parameters for filtering and access control
**Learn more:** [Generic Prompt Management API Documentation](../adding_provider/generic_prompt_management_api)
</TabItem>
</Tabs>
docs/my-website/docs/proxy/pyroscope_profiling.md
+43
View File
@@ -0,0 +1,43 @@
# Grafana Pyroscope CPU profiling
LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://grafana.com/docs/pyroscope/latest/) when enabled via environment variables. This is optional and off by default.
## Quick start
1. **Install the optional dependency** (required only when enabling Pyroscope):
```bash
pip install pyroscope-io
```
Or install the proxy extra:
```bash
pip install "litellm[proxy]"
```
2. **Set environment variables** before starting the proxy:
| Variable | Required | Description |
|----------|----------|-------------|
| `LITELLM_ENABLE_PYROSCOPE` | Yes (to enable) | Set to `true` to enable Pyroscope profiling. |
| `PYROSCOPE_APP_NAME` | Yes (when enabled) | Application name shown in the Pyroscope UI. |
| `PYROSCOPE_SERVER_ADDRESS` | Yes (when enabled) | Pyroscope server URL (e.g. `http://localhost:4040`). |
| `PYROSCOPE_SAMPLE_RATE` | No | Sample rate (integer). If unset, the pyroscope-io library default is used. |
3. **Start the proxy**; profiling will begin automatically when the proxy starts.
```bash
export LITELLM_ENABLE_PYROSCOPE=true
export PYROSCOPE_APP_NAME=litellm-proxy
export PYROSCOPE_SERVER_ADDRESS=http://localhost:4040
litellm --config config.yaml
```
4. **View profiles** in the Pyroscope (or Grafana) UI and select your `PYROSCOPE_APP_NAME`.
## Notes
- **Optional dependency**: `pyroscope-io` is an optional dependency. If it is not installed and `LITELLM_ENABLE_PYROSCOPE=true`, the proxy will log a warning and continue without profiling.
- **Platform support**: The `pyroscope-io` package uses a native extension and is not available on all platforms (e.g. Windows is excluded by the package).
- **Other settings**: See [Configuration settings](/proxy/config_settings) for all proxy environment variables.
docs/my-website/docs/proxy/release_cycle.md
+7 -1
View File
@@ -22,4 +22,10 @@ Stable releases come out every week (typically Sunday)
- 'patch' bumps: extremely minor addition that doesn't affect any existing functionality or add any user-facing features. (e.g. a 'created_at' column in a database table)
- 'minor' bumps: add a new feature or a new database table that is backward compatible.
- 'major' bumps: break backward compatibility.
- 'major' bumps: break backward compatibility.
### Enterprise Support
- Stable releases come out every week. Once a new one is available, we no longer provide support for an older one.
- If there is a MAJOR change (according to semvar conventions - e.g. 1.x.x -> 2.x.x), we can provide support for upto 90 days on the prior stable image.
docs/my-website/docs/proxy/request_headers.md
+4
View File
@@ -20,6 +20,10 @@ By default, LiteLLM does not forward client headers to LLM provider APIs. Howeve
`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata)
`x-litellm-customer-id`: Optional[str]: Standard header for passing a customer/end-user ID. Always checked without any configuration. [Learn More](./customers)
`x-litellm-end-user-id`: Optional[str]: Standard header for passing a customer/end-user ID. Always checked without any configuration. [Learn More](./customers)
## Anthropic Headers
`anthropic-version` Optional[str]: The version of the Anthropic API to use.
docs/my-website/docs/proxy/sync_anthropic_beta_headers.md
+128
View File
@@ -0,0 +1,128 @@
# Auto Sync Anthropic Beta Headers
Automatically keep your Anthropic beta headers configuration up to date without restarting your service. **This allows you to support new Anthropic beta features across all providers without restarting your service.**
## Overview
When Anthropic releases new beta features (e.g., new tool capabilities, extended context windows), you typically need to restart your LiteLLM service to get the latest beta header mappings for different providers (Anthropic, Bedrock, Vertex AI, Azure AI).
With auto-sync, LiteLLM automatically pulls the latest configuration from GitHub's [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) without requiring a restart. This means:
- **Zero downtime** when new beta features are released
- **Always up-to-date** provider support mappings
- **Automatic updates** - set it once and forget it
## Quick Start
**Manual sync:**
```bash
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
**Automatic sync every 24 hours:**
```bash
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
## API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/reload/anthropic_beta_headers` | POST | Manual sync |
| `/schedule/anthropic_beta_headers_reload?hours={hours}` | POST | Schedule periodic sync |
| `/schedule/anthropic_beta_headers_reload` | DELETE | Cancel scheduled sync |
| `/schedule/anthropic_beta_headers_reload/status` | GET | Check sync status |
**Authentication:** Requires admin role or master key
## Python Example
```python
import requests
def sync_anthropic_beta_headers(proxy_url, admin_token):
response = requests.post(
f"{proxy_url}/reload/anthropic_beta_headers",
headers={"Authorization": f"Bearer {admin_token}"}
)
return response.json()
# Usage
result = sync_anthropic_beta_headers("https://your-proxy-url", "your-admin-token")
print(result['message'])
```
## Configuration
**Custom beta headers config URL:**
```bash
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
```
**Use local beta headers config:**
```bash
export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
```
## Scheduling Automatic Reloads
Schedule automatic reloads to ensure your proxy always has the latest beta header mappings:
```bash
# Reload every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Check reload status:**
```bash
curl -X GET "https://your-proxy-url/schedule/anthropic_beta_headers_reload/status" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Response:**
```json
{
"scheduled": true,
"interval_hours": 24,
"last_run": "2026-02-13T10:00:00",
"next_run": "2026-02-14T10:00:00"
}
```
**Cancel scheduled reload:**
```bash
curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
## Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch beta headers config from | GitHub main branch |
| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
## How It Works
1. **Initial Load:** On startup, LiteLLM loads the beta headers configuration from the remote URL (or local file if configured)
2. **Caching:** The configuration is cached in memory to avoid repeated fetches on every request
3. **Scheduled Reload:** If configured, the proxy checks every 10 seconds whether it's time to reload based on your schedule
4. **Manual Reload:** You can trigger an immediate reload via the API endpoint
5. **Multi-Pod Support:** In multi-pod deployments, the reload configuration is stored in the database so all pods stay in sync
## Benefits
- **No Restarts Required:** Add support for new Anthropic beta features without downtime
- **Provider Compatibility:** Automatically get updated mappings for Bedrock, Vertex AI, Azure AI, etc.
- **Performance:** Configuration is cached and only reloaded when needed
- **Reliability:** Falls back to local configuration if remote fetch fails
## Related
- [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data
- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features
docs/my-website/docs/proxy/team_budgets.md
-1
View File
@@ -8,7 +8,6 @@ import TabItem from '@theme/TabItem';
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
## Default Budget for Auto-Generated JWT Teams
docs/my-website/docs/proxy/users.md
+1 -10
View File
@@ -68,13 +68,6 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
#### **Add budgets to teams**
```shell
@@ -822,12 +815,10 @@ Expected Response:
}
```
### [BETA] Multi-instance rate limiting
### Multi-instance rate limiting
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
docs/my-website/docs/proxy/virtual_keys.md
+6 -1
View File
@@ -549,11 +549,14 @@ curl 'http://localhost:4000/key/sk-1234/regenerate' \
"models": [
"gpt-4",
"gpt-3.5-turbo"
]
],
"grace_period": "48h"
}'
```
**Grace period (optional)**: Set `grace_period` (e.g. `"24h"`, `"2d"`, `"1w"`) to keep the old key valid for a transitional period. Both old and new keys work until the grace period elapses, enabling seamless cutover without production downtime. Omitted or empty = immediate revoke. Can also be set via `LITELLM_KEY_ROTATION_GRACE_PERIOD` env var for scheduled rotations.
**Read More**
- [Write rotated keys to secrets manager](https://docs.litellm.ai/docs/secret#aws-secret-manager)
@@ -640,11 +643,13 @@ Set these environment variables when starting the proxy:
|----------|-------------|---------|
| `LITELLM_KEY_ROTATION_ENABLED` | Enable the rotation worker | `false` |
| `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` | How often to scan for keys to rotate (in seconds) | `86400` (24 hours) |
| `LITELLM_KEY_ROTATION_GRACE_PERIOD` | Duration to keep old key valid after rotation (e.g. `24h`, `2d`) | `""` (immediate revoke) |
**Example:**
```bash
export LITELLM_KEY_ROTATION_ENABLED=true
export LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS=3600 # Check every hour
export LITELLM_KEY_ROTATION_GRACE_PERIOD=48h # Keep old key valid for 48h during cutover
litellm --config config.yaml
```
docs/my-website/docs/reasoning_content.md
+19
View File
@@ -642,6 +642,25 @@ model_list:
model: openai/responses/gpt-5-mini
```
**Per-model configuration** (recommended when using Open WebUI or clients that cannot set `extra_body`):
```yaml
model_list:
- model_name: gpt-5.1
litellm_params:
model: openai/gpt-5.1
# String format - uses reasoning_auto_summary for summary when set
reasoning_effort: "high"
model_info:
mode: responses # if using Responses API bridge
- model_name: gpt-5.1-with-summary
litellm_params:
model: openai/gpt-5.1
# Dict format - explicit control over effort and summary
reasoning_effort: {"effort": "high", "summary": "detailed"}
```
</TabItem>
</Tabs>
docs/my-website/docs/rerank.md
+24 -23
View File
@@ -8,15 +8,15 @@ LiteLLM Follows the [cohere api request / response for the rerank api](https://c
## Overview
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input query only (not documents) |
| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI | |
| Feature | Supported | Notes |
|---------|-----------------------------------------------------------------------------------------------------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input query only (not documents) |
| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI, watsonx.ai | |
## **LiteLLM Python SDK Usage**
### Quick Start
@@ -123,17 +123,18 @@ curl http://0.0.0.0:4000/rerank \
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Provider | Link to Usage |
|-------------|--------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
| Together AI| [Usage](../docs/providers/togetherai) |
| Azure AI| [Usage](../docs/providers/azure_ai#rerank-endpoint) |
| Jina AI| [Usage](../docs/providers/jina_ai) |
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |
| Vertex AI| [Usage](../docs/providers/vertex#rerank-api) |
| Fireworks AI| [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
| Voyage AI| [Usage](../docs/providers/voyage#rerank) |
| Provider | Link to Usage |
|--------------------------|------------------------------------------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
| Together AI | [Usage](../docs/providers/togetherai) |
| Azure AI | [Usage](../docs/providers/azure_ai#rerank-endpoint) |
| Jina AI | [Usage](../docs/providers/jina_ai) |
| AWS Bedrock | [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace | [Usage](../docs/providers/huggingface_rerank) |
| Infinity | [Usage](../docs/providers/infinity) |
| vLLM | [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra | [Usage](../docs/providers/deepinfra#rerank-endpoint) |
| Vertex AI | [Usage](../docs/providers/vertex#rerank-api) |
| Fireworks AI | [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
| Voyage AI | [Usage](../docs/providers/voyage#rerank) |
| IBM watsonx.ai | [Usage](../docs/providers/watsonx/rerank) |
docs/my-website/docs/response_api.md
+27 -8
View File
@@ -884,7 +884,13 @@ router = litellm.Router(
},
},
],
optional_pre_call_checks=["responses_api_deployment_check"],
# `responses_api_deployment_check` ensures Requests with `previous_response_id`
# are routed to the same deployment. `deployment_affinity` adds sticky sessions
# for requests without `previous_response_id` (useful for implicit caching).
# `session_affinity` adds sticky sessions based on `session_id` metadata.
optional_pre_call_checks=["responses_api_deployment_check", "deployment_affinity", "session_affinity"],
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds=3600,
)
# Initial request
@@ -911,7 +917,18 @@ follow_up = await router.aresponses(
#### 1. Setup session continuity on proxy config.yaml
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks: ["responses_api_deployment_check"]` in your proxy config.yaml.
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
- `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`)
- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
Notes:
- User-key affinity is keyed on `metadata.user_api_key_hash` (the API key hash). The OpenAI `user` request parameter is an end-user identifier and is intentionally not used for deployment affinity.
- Session-ID affinity is keyed on `metadata.session_id`. For proxy requests, this can be passed via the `x-litellm-session-id` HTTP header. For Python SDK requests, you can pass it via `litellm_metadata={"session_id": "value"}` in request args.
- `user_api_key_hash` is already SHA-256, and is used as-is (no double hashing).
- Affinity is scoped by a stable model identifier (the model-map key, e.g. `model_map_information.model_map_key`) so model aliases map to the same stickiness bucket.
- The mapping TTL is controlled by `deployment_affinity_ttl_seconds` (configured on Router init / proxy startup).
```yaml showLineNumbers title="config.yaml with Session Continuity"
model_list:
@@ -929,7 +946,12 @@ model_list:
api_base: https://endpoint2.openai.azure.com
router_settings:
optional_pre_call_checks: ["responses_api_deployment_check"]
optional_pre_call_checks:
- responses_api_deployment_check
- session_affinity
- deployment_affinity
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds: 3600
```
#### 2. Use the OpenAI Python SDK to make requests to LiteLLM Proxy
@@ -1029,6 +1051,8 @@ For long-running conversations, you can enable **server-side compaction** so tha
Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details.
> **Note:** You can use openai `context_management` format with Anthropic models via LiteLLM via responses API. LiteLLM will automatically translate this format for Anthropic and handle context management for you.
For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead.
### Python SDK
@@ -1356,8 +1380,3 @@ Response:
docs/my-website/docs/search/index.md
+1
View File
@@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.
docs/my-website/docs/troubleshoot.md
+12 -67
View File
@@ -1,102 +1,48 @@
# Troubleshooting & Support
## Information to Provide When Seeking Help
# Issue Reporting
When reporting issues, please include as much of the following as possible. It's okay if you can't provide everything—especially in production scenarios where the trigger might be unknown. Sharing most of this information will help us assist you more effectively.
### 1. LiteLLM Configuration File
## 1. LiteLLM Configuration File
Your `config.yaml` file (redact sensitive info like API keys). Include number of workers if not in config.
### 2. Initialization Command
## 2. Initialization Command
The command used to start LiteLLM (e.g., `litellm --config config.yaml --num_workers 8 --detailed_debug`).
### 3. LiteLLM Version
## 3. LiteLLM Version
- Current version
- Version when the issue first appeared (if different)
- Current version
- Version when the issue first appeared (if different)
- If upgraded, the version changed from → to
### 4. Environment Variables
## 4. Environment Variables
Non-sensitive environment variables not in your config (e.g., `NUM_WORKERS`, `LITELLM_LOG`, `LITELLM_MODE`). Do not include passwords or API keys.
### 5. Server Specifications
## 5. Server Specifications
CPU cores, RAM, OS, number of instances/replicas, etc.
### 6. Database and Redis Usage
## 6. Database and Redis Usage
- **Database:** Using database? (`DATABASE_URL` set), database type and version
- **Redis:** Using Redis? Redis version, configuration type (Standalone/Cluster/Sentinel).
### 7. Endpoints
## 7. Endpoints
The endpoint(s) you're using that are experiencing issues (e.g., `/chat/completions`, `/embeddings`).
### 8. Request Example
## 8. Request Example
A realistic example of the request causing issues, including expected vs. actual response and any error messages.
### 9. Error Logs, Stack Traces, and Metrics
## 9. Error Logs, Stack Traces, and Metrics
Full error logs, stack traces, and any images from service metrics (CPU, memory, request rates, etc.) that might help diagnose the issue.
---
## UI Issues
If you're experiencing issues with the LiteLLM Admin UI, please include the following information in addition to the general details above.
### 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
### 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
### 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
### 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
### 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
### 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
### 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
---
## Support Channels
[Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
@@ -109,4 +55,3 @@ Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
[![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw)
docs/my-website/docs/troubleshoot/latency_overhead.md
+90
View File
@@ -0,0 +1,90 @@
# Latency Overhead Troubleshooting
Use this guide when you see unexpected latency overhead between LiteLLM proxy and the LLM provider.
## Quick Checklist
1. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. Start here.
2. **Is DEBUG logging enabled?** This is the #1 cause of latency with large payloads.
3. **Are you sending large base64 payloads?** (images, PDFs) — see [Large Payload Overhead](#large-payload-overhead).
4. **Enable detailed timing headers** to pinpoint where time is spent.
## Diagnostic Headers
### `x-litellm-overhead-duration-ms` (always on)
Every response from LiteLLM includes this header. It shows the total latency overhead in milliseconds added by LiteLLM proxy (i.e. total response time minus the LLM API call time). Collect this on every request to understand your baseline overhead.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm-overhead-duration-ms
```
### `x-litellm-callback-duration-ms` (always on)
Shows time spent building callback/logging payloads (ms). If this is high (>100ms), your payloads may be too large for efficient logging.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm
```
### Detailed Timing Breakdown (opt-in)
Set `LITELLM_DETAILED_TIMING=true` to get per-phase timing in response headers:
| Header | What it measures |
|--------|-----------------|
| `x-litellm-timing-pre-processing-ms` | Auth, routing, request processing (before LLM call) |
| `x-litellm-timing-llm-api-ms` | Actual LLM API call duration |
| `x-litellm-timing-post-processing-ms` | Response processing (after LLM returns) |
| `x-litellm-timing-message-copy-ms` | Message copy time in logging layer |
```bash
# Enable detailed timing
export LITELLM_DETAILED_TIMING=true
```
## Large Payload Overhead
When sending large payloads (>1MB, e.g. base64-encoded images/PDFs), three things can add overhead:
### 1. DEBUG Logging (most common)
When `LITELLM_LOG=DEBUG` or `set_verbose=True` is enabled, every request payload is serialized with `json.dumps(indent=4)` synchronously. For a 2MB+ payload, this alone can take **2-5 seconds**.
**Fix:** Don't use DEBUG logging in production. Use `INFO` level instead:
```bash
export LITELLM_LOG=INFO
```
If you need DEBUG logging but have large payloads, you can increase the size threshold for full payload logging:
```bash
# Only fully serialize payloads under 100KB for DEBUG logs (default)
export MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG=102400
```
### 2. Base64 in Logging Payloads
Callback payloads (sent to Langfuse, etc.) include message content. Large base64 strings are automatically truncated to size placeholders in logging payloads.
You can control the truncation threshold:
```bash
# Max base64 characters before truncation (default: 64)
export MAX_BASE64_LENGTH_FOR_LOGGING=64
```
## Environment Variables Reference
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_DETAILED_TIMING` | `false` | Enable per-phase timing headers |
| `MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG` | `102400` | Max payload bytes for full DEBUG serialization |
| `MAX_BASE64_LENGTH_FOR_LOGGING` | `64` | Max base64 chars before truncation in logging |
docs/my-website/docs/troubleshoot/prisma_migrations.md
+6 -2
View File
@@ -2,6 +2,8 @@
Common Prisma migration issues encountered when upgrading or downgrading LiteLLM proxy versions, and how to fix them.
For a full guide on safely reverting your LiteLLM version, see the **[Safe Rollback Guide](rollback)**.
## How Prisma Migrations Work in LiteLLM
- LiteLLM uses [Prisma](https://www.prisma.io/) to manage its PostgreSQL database schema.
@@ -46,6 +48,8 @@ After deleting the entry, restart LiteLLM — it will re-apply the migration on
If deleting the migration entry and restarting doesn't resolve the issue, sync the schema directly:
> **Warning:** `prisma db push` can cause **data loss** if the Prisma schema removes columns or tables that exist in your database. Only use this as a last resort and ensure you have a database backup first.
```bash
DATABASE_URL="<your_database_url>" prisma db push
```
@@ -76,7 +80,7 @@ DELETE FROM "_prisma_migrations"
WHERE migration_name = '<failed_migration_name>';
```
3. If that doesn't work, use `prisma db push`:
3. If that doesn't work, use `prisma db push` (see [warning above](#step-2--if-that-doesnt-work-use-prisma-db-push) — back up your database first):
```bash
DATABASE_URL="<your_database_url>" prisma db push
@@ -106,7 +110,7 @@ LIMIT 20;
3. Restart LiteLLM to re-run migrations.
4. If that doesn't work, use `prisma db push`:
4. If that doesn't work, use `prisma db push` (see [warning above](#step-2--if-that-doesnt-work-use-prisma-db-push) — back up your database first):
```bash
DATABASE_URL="<your_database_url>" prisma db push
docs/my-website/docs/troubleshoot/rollback.md
+115
View File
@@ -0,0 +1,115 @@
# Safe Rollback Guide
This guide outlines the process for safely rolling back a LiteLLM Proxy deployment to a previous version.
We recommend rolling back to the previous [stable release](https://github.com/BerriAI/litellm/releases). Stable releases come out every week and follow the `main-v<VERSION>-stable` tag convention (e.g., `main-v1.77.2-stable`).
## 1. Determine Rollback Scope
Before proceeding, identify why you are rolling back:
- **Application Logic Error**: Reverting code changes but keeping the database schema.
- **Database Migration Failure**: Reverting changes that included database schema updates.
- **Performance Regression**: Reverting to a known stable version.
## 2. Back Up the Database
> **Always back up before rolling back.** Before making any changes, take a database snapshot or dump. This is your safety net if something goes wrong during the rollback.
```bash
# PostgreSQL example
pg_dump -h <host> -U <user> -d <database> -F c -f litellm_backup_$(date +%Y%m%d_%H%M%S).dump
```
If you are on a managed database (e.g., AWS RDS, GCP Cloud SQL), create a snapshot through your cloud console instead.
## 3. Pre-Rollback Checks
Before reverting, review these items:
- **`LITELLM_SALT_KEY`**: Do **not** change this value during rollback. It is used to encrypt/decrypt your LLM API Key credentials stored in the database. Changing it will make existing credentials unreadable. See [Best Practices for Production](../proxy/prod#8-set-litellm-salt-key).
- **`config.yaml`**: If you added settings specific to the newer version, the older version may not recognize them. Review your config and remove or comment out any settings that were introduced in the version you are rolling back from.
- **`DISABLE_SCHEMA_UPDATE`**: If you use the [Helm PreSync hook for migrations](../proxy/prod#7-use-helm-presync-hook-for-database-migrations-beta) with `DISABLE_SCHEMA_UPDATE=true` on your pods, migrations will **not** auto-run on restart. You will need to handle migration cleanup manually (see Step 5) or re-run the PreSync hook against the older chart version.
## 4. Revert Application Version
Revert your deployment to the previous stable Docker image or Helm chart version.
### Docker
Update your deployment manifest (e.g., K8s Deployment, Docker Compose) to use the previous version:
```yaml
# Example: Reverting to the previous stable release
image: docker.litellm.ai/berriai/litellm:main-v<VERSION>-stable
```
See [all available images](https://github.com/orgs/BerriAI/packages).
### Helm
If you deployed via Helm, use `helm rollback`:
```bash
helm rollback <release-name> [revision-number]
```
## 5. Handle Database Migrations
If you are rolling back to a version that did not have a specific migration, you may need to resolve the migration state in the database.
> LiteLLM uses `prisma migrate deploy` for production (enabled via `USE_PRISMA_MIGRATE=True`). If a migration partially failed or you are reverting code that expects an older schema, you need to clean up the migration history in the `_prisma_migrations` table. See [Best Practices for Production](../proxy/prod#9-use-prisma-migrate-deploy).
### Option A — Delete stale migration entries (recommended)
Connect to your PostgreSQL database and remove migration entries that belong to the version you are rolling back from. This lets LiteLLM re-apply them cleanly if you upgrade again later.
```sql
-- View recent migrations
SELECT migration_name, finished_at, rolled_back_at, logs
FROM "_prisma_migrations"
ORDER BY started_at DESC
LIMIT 10;
-- Delete migration entries from the version you are rolling back from
DELETE FROM "_prisma_migrations"
WHERE migration_name = '<migration_name_from_newer_version>';
```
After deleting the entries, restart LiteLLM — it will re-apply the correct migrations for its version on startup.
> **Note:** If you have `DISABLE_SCHEMA_UPDATE=true` set on your pods, migrations will not auto-run. You need to either temporarily set it to `false`, or re-run the Helm PreSync migration job targeting the older version.
### Option B — Use `prisma migrate resolve` (if you have CLI access)
If you have access to the Prisma CLI (e.g., in a local development environment or a debug container with the `litellm-proxy-extras` package installed):
```bash
DATABASE_URL="<your_database_url>" prisma migrate resolve --rolled-back "<migration_name>"
```
> **Note:** This requires the Prisma CLI to be available in your environment (installed via `prisma-client-py`). If you don't have CLI access (e.g., no shell into the running container), use **Option A** (direct SQL) instead.
### Auto-Recovery Logic
LiteLLM's internal `ProxyExtrasDBManager` automatically attempts to handle idempotent migrations. In many cases, simply rolling back the version and restarting the proxy will be enough if the database changes are additive (e.g., new columns or tables).
## 6. Verification Checklist
After rolling back, verify the health of the system:
- [ ] **Health Endpoint**: Confirm the `/health` endpoint returns `200 OK`.
- [ ] **Check Logs**: Ensure no Prisma errors appear — look for `relation "..." does not exist`, `column "..." does not exist`, or `prisma migrate` failures in the logs.
- [ ] **Spend Tracking**: Run a test completion and confirm the spend is recorded in the `LiteLLM_SpendLogs` table.
- [ ] **Billing (Lago)**: If using Lago for billing (e.g., Lago → Stripe), check proxy logs for `Logged Lago Object` to confirm usage events are being sent.
- [ ] **State Consistency**: If using Redis for caching or rate limiting, consider clearing the cache if the newer version changed the cache key structure.
- [ ] **Admin UI**: Verify the Admin UI loads and shows correct data for keys and teams.
## 7. Troubleshooting
### "New migrations cannot be applied"
If you see this error after a rollback, it means the database has a migration in a "failed" state.
1. Identify the failed migration name (see the SQL query in Step 5).
2. Delete the failed entry from `_prisma_migrations`.
3. Restart the proxy.
### "relation X does not exist"
This typically means a migration entry exists in `_prisma_migrations` but the actual table/column was never created or was dropped.
1. Delete the stale migration entry.
2. Restart LiteLLM so it re-runs the migration.
For more details on Prisma errors, see [Prisma Migrations Troubleshoot](prisma_migrations).
docs/my-website/docs/troubleshoot/ui_issues.md
+49
View File
@@ -0,0 +1,49 @@
# UI Troubleshooting
If you're experiencing issues with the LiteLLM Admin UI, please include the following information when reporting.
## 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
## 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
## 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
## 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
## 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
## 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
## 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
docs/my-website/docs/tutorials/claude_code_beta_headers.md
+52 -3
View File
@@ -92,9 +92,34 @@ Open `anthropic_beta_headers_config.json` and add the new header to each provide
- **Header transformations**: Some providers use different header names (e.g., Bedrock maps `advanced-tool-use-2025-11-20` to `tool-search-tool-2025-10-19`)
- **Alphabetical order**: Keep headers sorted alphabetically for maintainability
### Step 3: Restart Your Application
### Step 3: Reload Configuration (No Restart Required!)
After updating the config file, restart your LiteLLM proxy or application:
**Option 1: Dynamic Reload Without Restart**
Instead of restarting your application, you can dynamically reload the beta headers configuration using environment variables and API endpoints:
```bash
# Set environment variable to fetch from remote URL (Do this if you want to point it to some other URL)
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
# Manually trigger reload via API (no restart needed!)
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Option 2: Schedule Automatic Reloads**
Set up automatic reloading to always stay up-to-date with the latest beta headers:
```bash
# Reload configuration every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Option 3: Traditional Restart**
If you prefer the traditional approach, restart your LiteLLM proxy or application:
```bash
# If using LiteLLM proxy
@@ -104,7 +129,11 @@ litellm --config config.yaml
# Just restart your Python application
```
The updated configuration will be loaded automatically.
:::tip Zero-Downtime Updates
With dynamic reloading, you can fix invalid beta header errors **without restarting your service**! This is especially useful in production environments where downtime is costly.
See [Auto Sync Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) for complete documentation.
:::
## Fixing Invalid Beta Header Errors
@@ -215,6 +244,26 @@ Result sent to Bedrock:
anthropic-beta: computer-use-2025-01-24
```
## Dynamic Configuration Management (No Restart Required!)
### Environment Variables
Control how LiteLLM loads the beta headers configuration:
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch config from | GitHub main branch |
| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
**Example: Use Custom Config URL**
```bash
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://your-company.com/custom-beta-headers.json"
```
**Example: Use Local Config Only (No Remote Fetching)**
```bash
export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
```
## Provider-Specific Notes
### Bedrock
docs/my-website/docs/tutorials/claude_mcp.md
+54 -18
View File
@@ -9,7 +9,7 @@ Note: LiteLLM supports OAuth for MCP servers as well. [Learn more](https://docs.
## Connecting MCP Servers
You can also connect MCP servers to Claude Code via LiteLLM Proxy.
You can connect MCP servers to Claude Code via LiteLLM Proxy.
1. Add the MCP server to your `config.yaml`
@@ -23,6 +23,7 @@ In this example, we'll add the Github MCP server to our `config.yaml`
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
transport: "http"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
@@ -34,31 +35,70 @@ mcp_servers:
In this example, we'll add the Atlassian MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
atlassian_mcp:
server_id: atlassian_mcp_id
url: "https://mcp.atlassian.com/v1/sse"
transport: "sse"
auth_type: oauth2
mcp_servers:
atlassian_mcp:
url: "https://mcp.atlassian.com/v1/mcp"
transport: "http"
auth_type: oauth2
```
</TabItem>
</Tabs>
:::important
The server name under `mcp_servers:` (e.g. `atlassian_mcp`, `github_mcp`) **must match** the name used in the Claude Code URL path (`/mcp/<server_name>`). A mismatch will cause a 404 error during OAuth.
:::
2. Start LiteLLM Proxy
Since Claude Code needs a publicly accessible URL for the OAuth callback, expose your proxy via ngrok or a similar tool.
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Use the MCP server in Claude Code
```bash
claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
# In a separate terminal — expose proxy for OAuth callbacks
ngrok http 4000
```
For MCP servers that require dynamic client registration (such as Atlassian), please set `x-litellm-api-key: Bearer sk-LITELLM_VIRTUAL_KEY` instead of using `Authorization: Bearer LITELLM_VIRTUAL_KEY`.
3. Add the MCP server to Claude Code
<Tabs>
<TabItem value="github" label="GitHub MCP">
```bash
claude mcp add --transport http litellm-github https://your-ngrok-url.ngrok-free.dev/mcp/github_mcp \
--header "x-litellm-api-key: Bearer sk-1234"
```
</TabItem>
<TabItem value="atlassian" label="Atlassian MCP">
```bash
claude mcp add --transport http litellm-atlassian https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp \
--header "x-litellm-api-key: Bearer sk-1234"
```
</TabItem>
</Tabs>
**Parameter breakdown:**
| Parameter | Description |
|-----------|-------------|
| `--transport http` | Use HTTP transport for the MCP connection |
| `litellm-atlassian` | The name for this MCP server **on Claude Code** — can be anything you choose |
| `https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp` | The LiteLLM proxy URL. Format: `<PROXY_URL>/mcp/<server_name_on_litellm>`. The `atlassian_mcp` part **must match** the key under `mcp_servers:` in your LiteLLM proxy config |
| `--header "x-litellm-api-key: Bearer sk-1234"` | Your LiteLLM virtual key for authentication to the proxy |
You can also add the MCP server directly to your `~/.claude.json` file instead of using `claude mcp add`. [See Claude Code docs](https://docs.anthropic.com/en/docs/claude-code/mcp).
:::note
For MCP servers that require OAuth (such as Atlassian), use `x-litellm-api-key` instead of `Authorization` for the LiteLLM virtual key. The `Authorization` header is reserved for the OAuth flow.
:::
4. Authenticate via Claude Code
@@ -68,24 +108,20 @@ a. Start Claude Code
claude
```
b. Authenticate via Claude Code
b. Open the MCP menu
```bash
/mcp
```
c. Select the MCP server
c. Select the MCP server (e.g. `litellm-atlassian`)
```bash
> litellm_proxy
```
d. Start Oauth flow via Claude Code
d. Start the OAuth flow
```bash
> 1. Authenticate
2. Reconnect
3. Disable
3. Disable
```
e. Once completed, you should see this success message:
docs/my-website/img/passthrough_method_setup.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 77 KiB

docs/my-website/img/passthrough_query_default.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 84 KiB

docs/my-website/img/project_spend.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 850 KiB

docs/my-website/img/release_notes/guard_actions.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 506 KiB

docs/my-website/img/ui_access_groups.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 338 KiB

docs/my-website/package-lock.json
Generated
+89 -77
View File
@@ -8339,9 +8339,9 @@
}
},
"node_modules/ajv": {
"version": "8.17.1",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
"integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.3",
@@ -8721,10 +8721,13 @@
}
},
"node_modules/balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
"license": "MIT"
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.3.tgz",
"integrity": "sha512-1pHv8LX9CpKut1Zp4EXey7Z8OfH11ONNH6Dhi2WDUt31VVZFXZzKwXcysBgqSumFCmR+0dqjMK5v5JiFHzi0+g==",
"license": "MIT",
"engines": {
"node": "20 || >=22"
}
},
"node_modules/bare-events": {
"version": "2.8.2",
@@ -8838,12 +8841,15 @@
"license": "MIT"
},
"node_modules/baseline-browser-mapping": {
"version": "2.8.30",
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.30.tgz",
"integrity": "sha512-aTUKW4ptQhS64+v2d6IkPzymEzzhw+G0bA1g3uBRV3+ntkH+svttKseW5IOR4Ed6NUVKqnY7qT3dKvzQ7io4AA==",
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.0.tgz",
"integrity": "sha512-lIyg0szRfYbiy67j9KN8IyeD7q7hcmqnJ1ddWmNt19ItGpNN64mnllmxUNFIOdOm6by97jlL6wfpTTJrmnjWAA==",
"license": "Apache-2.0",
"bin": {
"baseline-browser-mapping": "dist/cli.js"
"baseline-browser-mapping": "dist/cli.cjs"
},
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/batch": {
@@ -9024,13 +9030,15 @@
}
},
"node_modules/brace-expansion": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz",
"integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
"balanced-match": "^4.0.2"
},
"engines": {
"node": "20 || >=22"
}
},
"node_modules/braces": {
@@ -9046,9 +9054,9 @@
}
},
"node_modules/browserslist": {
"version": "4.28.0",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz",
"integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==",
"version": "4.28.1",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz",
"integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==",
"funding": [
{
"type": "opencollective",
@@ -9065,11 +9073,11 @@
],
"license": "MIT",
"dependencies": {
"baseline-browser-mapping": "^2.8.25",
"caniuse-lite": "^1.0.30001754",
"electron-to-chromium": "^1.5.249",
"baseline-browser-mapping": "^2.9.0",
"caniuse-lite": "^1.0.30001759",
"electron-to-chromium": "^1.5.263",
"node-releases": "^2.0.27",
"update-browserslist-db": "^1.1.4"
"update-browserslist-db": "^1.2.0"
},
"bin": {
"browserslist": "cli.js"
@@ -9262,9 +9270,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001756",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001756.tgz",
"integrity": "sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==",
"version": "1.0.30001770",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001770.tgz",
"integrity": "sha512-x/2CLQ1jHENRbHg5PSId2sXq1CIO1CISvwWAj027ltMVG2UNgW+w9oH2+HzgEIRFembL8bUlXtfbBHR1fCg2xw==",
"funding": [
{
"type": "opencollective",
@@ -9764,12 +9772,6 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"license": "MIT"
},
"node_modules/confbox": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.2.tgz",
@@ -11413,9 +11415,9 @@
"license": "MIT"
},
"node_modules/electron-to-chromium": {
"version": "1.5.259",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.259.tgz",
"integrity": "sha512-I+oLXgpEJzD6Cwuwt1gYjxsDmu/S/Kd41mmLA3O+/uH2pFRO/DvOjUyGozL8j3KeLV6WyZ7ssPwELMsXCcsJAQ==",
"version": "1.5.302",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.302.tgz",
"integrity": "sha512-sM6HAN2LyK82IyPBpznDRqlTQAtuSaO+ShzFiWTvoMJLHyZ+Y39r8VMfHzwbU8MVBzQ4Wdn85+wlZl2TLGIlwg==",
"license": "ISC"
},
"node_modules/emoji-regex": {
@@ -11468,13 +11470,13 @@
}
},
"node_modules/enhanced-resolve": {
"version": "5.18.3",
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.3.tgz",
"integrity": "sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==",
"version": "5.19.0",
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz",
"integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.2.4",
"tapable": "^2.2.0"
"tapable": "^2.3.0"
},
"engines": {
"node": ">=10.13.0"
@@ -11520,9 +11522,9 @@
}
},
"node_modules/es-module-lexer": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz",
"integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz",
"integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==",
"license": "MIT"
},
"node_modules/es-object-atoms": {
@@ -12142,9 +12144,9 @@
}
},
"node_modules/file-loader/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
@@ -16720,15 +16722,18 @@
"license": "ISC"
},
"node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"license": "ISC",
"version": "10.2.1",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.1.tgz",
"integrity": "sha512-MClCe8IL5nRRmawL6ib/eT4oLyeKMGCghibcDWK+J0hh0Q8kqSdia6BvbRMVk6mPa6WqUa5uR2oxt6C5jd533A==",
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^1.1.7"
"brace-expansion": "^5.0.2"
},
"engines": {
"node": "*"
"node": "20 || >=22"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/minimist": {
@@ -17021,9 +17026,9 @@
}
},
"node_modules/null-loader/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
@@ -19294,9 +19299,9 @@
}
},
"node_modules/qs": {
"version": "6.14.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz",
"integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==",
"version": "6.14.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz",
"integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==",
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.1.0"
@@ -20455,6 +20460,13 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",
@@ -21464,9 +21476,9 @@
}
},
"node_modules/terser-webpack-plugin": {
"version": "5.3.14",
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz",
"integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==",
"version": "5.3.16",
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz",
"integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==",
"license": "MIT",
"dependencies": {
"@jridgewell/trace-mapping": "^0.3.25",
@@ -21921,9 +21933,9 @@
}
},
"node_modules/update-browserslist-db": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz",
"integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==",
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
"integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==",
"funding": [
{
"type": "opencollective",
@@ -22061,9 +22073,9 @@
}
},
"node_modules/url-loader/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
@@ -22365,9 +22377,9 @@
"license": "MIT"
},
"node_modules/watchpack": {
"version": "2.4.4",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.4.tgz",
"integrity": "sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==",
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz",
"integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==",
"license": "MIT",
"dependencies": {
"glob-to-regexp": "^0.4.1",
@@ -22412,9 +22424,9 @@
"license": "BSD-2-Clause"
},
"node_modules/webpack": {
"version": "5.103.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.103.0.tgz",
"integrity": "sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==",
"version": "5.105.2",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.2.tgz",
"integrity": "sha512-dRXm0a2qcHPUBEzVk8uph0xWSjV/xZxenQQbLwnwP7caQCYpqG1qddwlyEkIDkYn0K8tvmcrZ+bOrzoQ3HxCDw==",
"license": "MIT",
"dependencies": {
"@types/eslint-scope": "^3.7.7",
@@ -22425,10 +22437,10 @@
"@webassemblyjs/wasm-parser": "^1.14.1",
"acorn": "^8.15.0",
"acorn-import-phases": "^1.0.3",
"browserslist": "^4.26.3",
"browserslist": "^4.28.1",
"chrome-trace-event": "^1.0.2",
"enhanced-resolve": "^5.17.3",
"es-module-lexer": "^1.2.1",
"enhanced-resolve": "^5.19.0",
"es-module-lexer": "^2.0.0",
"eslint-scope": "5.1.1",
"events": "^3.2.0",
"glob-to-regexp": "^0.4.1",
@@ -22439,8 +22451,8 @@
"neo-async": "^2.6.2",
"schema-utils": "^4.3.3",
"tapable": "^2.3.0",
"terser-webpack-plugin": "^5.3.11",
"watchpack": "^2.4.4",
"terser-webpack-plugin": "^5.3.16",
"watchpack": "^2.5.1",
"webpack-sources": "^3.3.3"
},
"bin": {
docs/my-website/package.json
+18 -2
View File
@@ -61,10 +61,26 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.7",
"tar": ">=7.5.8",
"@isaacs/brace-expansion": ">=5.0.1",
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1",
"lodash-es": ">=4.17.23"
"lodash-es": ">=4.17.23",
"schema-utils@3": {
"ajv": "6.14.0"
},
"schema-utils@4": {
"ajv": "8.18.0"
},
"file-loader": {
"ajv": "6.14.0"
},
"null-loader": {
"ajv": "6.14.0"
},
"url-loader": {
"ajv": "6.14.0"
},
"minimatch": "10.2.1"
}
}
docs/my-website/release_notes/v1.81.12.md
+433
View File
@@ -0,0 +1,433 @@
---
title: "v1.81.12-stable - Guardrail Policy Templates & Action Builder"
slug: "v1-81-12"
date: 2026-02-14T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.12-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.12
```
</TabItem>
</Tabs>
## Key Highlights
- **Policy Templates** - [Pre-configured guardrail policy templates for common safety and compliance use-cases (including NSFW, toxic content, and child safety)](../../docs/proxy/guardrails/policy_templates)
- **Guardrail Action Builder** - [Build and customize guardrail policy flows with the new action-builder UI and conditional execution support](../../docs/proxy/guardrails/policy_templates)
- **MCP OAuth2 M2M + Tracing** - [Add machine-to-machine OAuth2 support for MCP servers and OpenTelemetry tracing for MCP calls through AI Gateway](../../docs/mcp)
- **Responses API `shell` Tool & `context_management` support** - [Server-side context management (compaction) and Shell tool support for the OpenAI Responses API](../../docs/response_api)
- **Access Groups** - [Create access groups to manage model, MCP server, and agent access across teams and keys](../../docs/proxy/access_groups)
- **50+ New Bedrock Regional Model Entries** - DeepSeek V3.2, MiniMax M2.1, Kimi K2.5, Qwen3 Coder Next, and NVIDIA Nemotron Nano across multiple regions
- **Add Semgrep & fix OOMs** - [Static analysis rules and out-of-memory fixes](#add-semgrep--fix-ooms) - [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
---
## Add Semgrep & fix OOMs
This release fixes out-of-memory (OOM) risks from unbounded `asyncio.Queue()` usage. Log queues (e.g. GCS bucket) and DB spend-update queues were previously unbounded and could grow without limit under load. They now use a configurable max size (`LITELLM_ASYNCIO_QUEUE_MAXSIZE`, default 1000); when full, queues flush immediately to make room instead of growing memory. A Semgrep rule (`.semgrep/rules/python/unbounded-memory.yml`) was added to flag similar unbounded-memory patterns in future code. [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
---
## Guardrail Action Builder
This release adds a visual action builder for guardrail policies with conditional execution support. You can now chain guardrails into multi-step pipelines — if a simple guardrail fails, route to an advanced one instead of immediately blocking. Each step has configurable ON PASS and ON FAIL actions (Next Step, Block, or Allow), and you can test the full pipeline with a sample message before saving.
![Guardrail Action Builder](../img/release_notes/guard_actions.png)
### Access Groups
Access Groups simplify defining resource access across your organization. One group can grant access to models, MCP servers, and agents—simply attach it to a key or team. Create groups in the Admin UI, define which resources each group includes, then assign the group when creating keys or teams. Updates to a group apply automatically to all attached keys and teams.
<Image img={require('../img/ui_access_groups.png')} />
## New Providers and Endpoints
### New Providers (2 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | --------------------------- | ----------- |
| [Scaleway](../../docs/providers/scaleway) | `/chat/completions` | Scaleway Generative APIs for chat completions |
| [Sarvam AI](../../docs/providers/sarvam) | `/chat/completions`, `/audio/transcriptions`, `/audio/speech` | Sarvam AI STT and TTS support for Indian languages |
---
## New Models / Updated Models
#### New Model Support (19 highlighted models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) |
| -------- | ----- | -------------- | ------------------- | -------------------- |
| AWS Bedrock | `deepseek.v3.2` | 164K | $0.62 | $1.85 |
| AWS Bedrock | `minimax.minimax-m2.1` | 196K | $0.30 | $1.20 |
| AWS Bedrock | `moonshotai.kimi-k2.5` | 262K | $0.60 | $3.00 |
| AWS Bedrock | `moonshotai.kimi-k2-thinking` | 262K | $0.73 | $3.03 |
| AWS Bedrock | `qwen.qwen3-coder-next` | 262K | $0.50 | $1.20 |
| AWS Bedrock | `nvidia.nemotron-nano-3-30b` | 262K | $0.06 | $0.24 |
| Azure AI | `azure_ai/kimi-k2.5` | 262K | $0.60 | $3.00 |
| Vertex AI | `vertex_ai/zai-org/glm-5-maas` | 200K | $1.00 | $3.20 |
| MiniMax | `minimax/MiniMax-M2.5` | 1M | $0.30 | $1.20 |
| MiniMax | `minimax/MiniMax-M2.5-lightning` | 1M | $0.30 | $2.40 |
| Dashscope | `dashscope/qwen3-max` | 258K | Tiered pricing | Tiered pricing |
| Perplexity | `perplexity/preset/pro-search` | - | Per-request | Per-request |
| Perplexity | `perplexity/openai/gpt-4o` | - | Per-request | Per-request |
| Perplexity | `perplexity/openai/gpt-5.2` | - | Per-request | Per-request |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-opus-4.6` | 200K | $5.00 | $25.00 |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-sonnet-4` | 200K | $3.00 | $15.00 |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-haiku-4.5` | 200K | $1.00 | $5.00 |
| Sarvam AI | `sarvam/sarvam-m` | 8K | Free tier | Free tier |
| Anthropic | `fast/claude-opus-4-6` | 1M | $30.00 | $150.00 |
*Note: AWS Bedrock models are available across multiple regions (us-east-1, us-east-2, us-west-2, eu-central-1, eu-north-1, ap-northeast-1, ap-south-1, ap-southeast-3, sa-east-1). 54 regional model entries were added in total.*
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Enable non-tool structured outputs on Claude Opus 4.5 and 4.6 using `output_format` param - [PR #20548](https://github.com/BerriAI/litellm/pull/20548)
- Add support for `anthropic_messages` call type in prompt caching - [PR #19233](https://github.com/BerriAI/litellm/pull/19233)
- Managing Anthropic Beta Headers with remote URL fetching - [PR #20935](https://github.com/BerriAI/litellm/pull/20935), [PR #21110](https://github.com/BerriAI/litellm/pull/21110)
- Remove `x-anthropic-billing` block - [PR #20951](https://github.com/BerriAI/litellm/pull/20951)
- Use Authorization Bearer for OAuth tokens instead of x-api-key - [PR #21039](https://github.com/BerriAI/litellm/pull/21039)
- Filter unsupported JSON schema constraints for structured outputs - [PR #20813](https://github.com/BerriAI/litellm/pull/20813)
- New Claude Opus 4.6 features for `/v1/messages` - [PR #20733](https://github.com/BerriAI/litellm/pull/20733)
- Fix `reasoning_effort=None` and `"none"` should return None for Opus 4.6 - [PR #20800](https://github.com/BerriAI/litellm/pull/20800)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Extend model support with 4 new beta models - [PR #21035](https://github.com/BerriAI/litellm/pull/21035)
- Add Claude Opus 4.6 to `_supports_tool_search_on_bedrock` - [PR #21017](https://github.com/BerriAI/litellm/pull/21017)
- Correct Bedrock Claude Opus 4.6 model IDs (remove `:0` suffix) - [PR #20564](https://github.com/BerriAI/litellm/pull/20564), [PR #20671](https://github.com/BerriAI/litellm/pull/20671)
- Add `output_config` as supported param - [PR #20748](https://github.com/BerriAI/litellm/pull/20748)
- **[Vertex AI](../../docs/providers/vertex)**
- Add Vertex GLM-5 model support - [PR #21053](https://github.com/BerriAI/litellm/pull/21053)
- Propagate `extra_headers` anthropic-beta to request body - [PR #20666](https://github.com/BerriAI/litellm/pull/20666)
- Preserve `usageMetadata` in `_hidden_params` - [PR #20559](https://github.com/BerriAI/litellm/pull/20559)
- Map `IMAGE_PROHIBITED_CONTENT` to `content_filter` - [PR #20524](https://github.com/BerriAI/litellm/pull/20524)
- Add RAG ingest for Vertex AI - [PR #21120](https://github.com/BerriAI/litellm/pull/21120)
- **[OCI / Cohere](../../docs/providers/cohere)**
- OCI Cohere responseFormat/Pydantic support - [PR #20663](https://github.com/BerriAI/litellm/pull/20663)
- Fix OCI Cohere system messages by populating `preambleOverride` - [PR #20958](https://github.com/BerriAI/litellm/pull/20958)
- **[Perplexity](../../docs/providers/perplexity)**
- Perplexity Research API support with preset search - [PR #20860](https://github.com/BerriAI/litellm/pull/20860)
- **[MiniMax](../../docs/providers/minimax)**
- Add MiniMax-M2.5 and MiniMax-M2.5-lightning models - [PR #21054](https://github.com/BerriAI/litellm/pull/21054)
- **[Kimi / Moonshot](../../docs/providers/moonshot)**
- Add Kimi model pricing by region - [PR #20855](https://github.com/BerriAI/litellm/pull/20855)
- Add `moonshotai.kimi-k2.5` - [PR #20863](https://github.com/BerriAI/litellm/pull/20863)
- **[Dashscope](../../docs/providers/dashscope)**
- Add `dashscope/qwen3-max` model with tiered pricing - [PR #20919](https://github.com/BerriAI/litellm/pull/20919)
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- Add new Vercel AI Anthropic models - [PR #20745](https://github.com/BerriAI/litellm/pull/20745)
- **[Azure AI](../../docs/providers/azure_ai)**
- Add `azure_ai/kimi-k2.5` to Azure model DB - [PR #20896](https://github.com/BerriAI/litellm/pull/20896)
- Support Azure AD token auth for non-Claude azure_ai models - [PR #20981](https://github.com/BerriAI/litellm/pull/20981)
- Fix Azure batches issues - [PR #21092](https://github.com/BerriAI/litellm/pull/21092)
- **[DeepSeek](../../docs/providers/deepseek)**
- Sync DeepSeek model metadata and add bare-name fallback - [PR #20938](https://github.com/BerriAI/litellm/pull/20938)
- **[Gemini](../../docs/providers/gemini)**
- Handle image in assistant message for Gemini - [PR #20845](https://github.com/BerriAI/litellm/pull/20845)
- Add missing tpm/rpm for Gemini models - [PR #21175](https://github.com/BerriAI/litellm/pull/21175)
- **General**
- Add 30 missing models to pricing JSON - [PR #20797](https://github.com/BerriAI/litellm/pull/20797)
- Cleanup 39 deprecated OpenRouter models - [PR #20786](https://github.com/BerriAI/litellm/pull/20786)
- Standardize endpoint `display_name` naming convention - [PR #20791](https://github.com/BerriAI/litellm/pull/20791)
- Fix and stabilize model cost map formatting - [PR #20895](https://github.com/BerriAI/litellm/pull/20895)
- Export `PermissionDeniedError` from `litellm.__init__` - [PR #20960](https://github.com/BerriAI/litellm/pull/20960)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix `get_supported_anthropic_messages_params` - [PR #20752](https://github.com/BerriAI/litellm/pull/20752)
- Fix `base_model` name for body and deployment name in URL - [PR #20747](https://github.com/BerriAI/litellm/pull/20747)
- **[Azure](../../docs/providers/azure/azure)**
- Preserve `content_policy_violation` error details from Azure OpenAI - [PR #20883](https://github.com/BerriAI/litellm/pull/20883)
- **[Vertex AI](../../docs/providers/vertex)**
- Fix Gemini multi-turn tool calling message formatting (added and reverted) - [PR #20569](https://github.com/BerriAI/litellm/pull/20569), [PR #21051](https://github.com/BerriAI/litellm/pull/21051)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Add server-side context management (compaction) support - [PR #21058](https://github.com/BerriAI/litellm/pull/21058)
- Add Shell tool support for OpenAI Responses API - [PR #21063](https://github.com/BerriAI/litellm/pull/21063)
- Preserve tool call argument deltas when streaming id is omitted - [PR #20712](https://github.com/BerriAI/litellm/pull/20712)
- Preserve interleaved thinking/redacted_thinking blocks during streaming - [PR #20702](https://github.com/BerriAI/litellm/pull/20702)
- **[Chat Completions](../../docs/completion/input)**
- Add Web Search support using LiteLLM `/search` (web search interception hook) - [PR #20483](https://github.com/BerriAI/litellm/pull/20483)
- Preserved nullable object fields by carrying schema properties - [PR #19132](https://github.com/BerriAI/litellm/pull/19132)
- Support `prompt_cache_key` for OpenAI and Azure chat completions - [PR #20989](https://github.com/BerriAI/litellm/pull/20989)
- **[Pass-Through Endpoints](../../docs/pass_through/bedrock)**
- Add support for `langchain_aws` via LiteLLM passthrough - [PR #20843](https://github.com/BerriAI/litellm/pull/20843)
- Add `custom_body` parameter to `endpoint_func` in `create_pass_through_route` - [PR #20849](https://github.com/BerriAI/litellm/pull/20849)
- **[Vector Stores](../../docs/providers/openai)**
- Add `target_model_names` for vector store endpoints - [PR #21089](https://github.com/BerriAI/litellm/pull/21089)
- **General**
- Add `output_config` as supported param - [PR #20748](https://github.com/BerriAI/litellm/pull/20748)
- Add managed error file support - [PR #20838](https://github.com/BerriAI/litellm/pull/20838)
#### Bugs
- **General**
- Stop leaking Python tracebacks in streaming SSE error responses - [PR #20850](https://github.com/BerriAI/litellm/pull/20850)
- Fix video list pagination cursors not encoded with provider metadata - [PR #20710](https://github.com/BerriAI/litellm/pull/20710)
- Handle `metadata=None` in SDK path retry/error logic - [PR #20873](https://github.com/BerriAI/litellm/pull/20873)
- Fix Spend logs pickle error with Pydantic models and redaction - [PR #20685](https://github.com/BerriAI/litellm/pull/20685)
- Remove duplicate `PerplexityResponsesConfig` from `LLM_CONFIG_NAMES` - [PR #21105](https://github.com/BerriAI/litellm/pull/21105)
---
## Management Endpoints / UI
#### Features
- **Access Groups**
- New Access Groups feature for managing model, MCP server, and agent access - [PR #21022](https://github.com/BerriAI/litellm/pull/21022)
- Access Groups table and details page UI - [PR #21165](https://github.com/BerriAI/litellm/pull/21165)
- Refactor `model_ids` to `model_names` for backwards compatibility - [PR #21166](https://github.com/BerriAI/litellm/pull/21166)
- **Policies**
- Allow connecting Policies to Tags, simulating Policies, viewing key/team counts - [PR #20904](https://github.com/BerriAI/litellm/pull/20904)
- Guardrail pipeline support for conditional sequential execution - [PR #21177](https://github.com/BerriAI/litellm/pull/21177)
- Pipeline flow builder UI for guardrail policies - [PR #21188](https://github.com/BerriAI/litellm/pull/21188)
- **SSO / Auth**
- New Login With SSO Button - [PR #20908](https://github.com/BerriAI/litellm/pull/20908)
- M2M OAuth2 UI Flow - [PR #20794](https://github.com/BerriAI/litellm/pull/20794)
- Allow Organization and Team Admins to call `/invitation/new` - [PR #20987](https://github.com/BerriAI/litellm/pull/20987)
- Invite User: Email Integration Alert - [PR #20790](https://github.com/BerriAI/litellm/pull/20790)
- Populate identity fields in proxy admin JWT early-return path - [PR #21169](https://github.com/BerriAI/litellm/pull/21169)
- **Spend Logs**
- Show predefined error codes in filter with user definable fallback - [PR #20773](https://github.com/BerriAI/litellm/pull/20773)
- Paginated searchable model select - [PR #20892](https://github.com/BerriAI/litellm/pull/20892)
- Sorting columns support - [PR #21143](https://github.com/BerriAI/litellm/pull/21143)
- Allow sorting on `/spend/logs/ui` - [PR #20991](https://github.com/BerriAI/litellm/pull/20991)
- **UI Improvements**
- Navbar: Option to hide Usage Popup - [PR #20910](https://github.com/BerriAI/litellm/pull/20910)
- Model Page: Improve Credentials Messaging - [PR #21076](https://github.com/BerriAI/litellm/pull/21076)
- Fallbacks: Default configurable to 10 models - [PR #21144](https://github.com/BerriAI/litellm/pull/21144)
- Fallback display with arrows and card structure - [PR #20922](https://github.com/BerriAI/litellm/pull/20922)
- Team Info: Migrate to AntD Tabs + Table - [PR #20785](https://github.com/BerriAI/litellm/pull/20785)
- AntD refactoring and 0 cost models fix - [PR #20687](https://github.com/BerriAI/litellm/pull/20687)
- Zscaler AI Guard UI - [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
- Include Config Defined Pass Through Endpoints - [PR #20898](https://github.com/BerriAI/litellm/pull/20898)
- Rename "HTTP" to "Streamable HTTP (Recommended)" in MCP server page - [PR #21000](https://github.com/BerriAI/litellm/pull/21000)
- MCP server discovery UI - [PR #21079](https://github.com/BerriAI/litellm/pull/21079)
- **Virtual Keys**
- Allow Management keys to access `user/daily/activity` and team - [PR #20124](https://github.com/BerriAI/litellm/pull/20124)
- Skip premium check for empty metadata fields on team/key update - [PR #20598](https://github.com/BerriAI/litellm/pull/20598)
#### Bugs
- Logs: Fix Input and Output Copying - [PR #20657](https://github.com/BerriAI/litellm/pull/20657)
- Teams: Fix Available Teams - [PR #20682](https://github.com/BerriAI/litellm/pull/20682)
- Spend Logs: Reset Filters Resets Custom Date Range - [PR #21149](https://github.com/BerriAI/litellm/pull/21149)
- Usage: Request Chart stack variant fix - [PR #20894](https://github.com/BerriAI/litellm/pull/20894)
- Add Auto Router: Description Text Input Focus - [PR #21004](https://github.com/BerriAI/litellm/pull/21004)
- Guardrail Edit: LiteLLM Content Filter Categories - [PR #21002](https://github.com/BerriAI/litellm/pull/21002)
- Add null guard for models in API keys table - [PR #20655](https://github.com/BerriAI/litellm/pull/20655)
- Show error details instead of 'Data Not Available' for failed requests - [PR #20656](https://github.com/BerriAI/litellm/pull/20656)
- Fix Spend Management Tests - [PR #21088](https://github.com/BerriAI/litellm/pull/21088)
- Fix JWT email domain validation error message - [PR #21212](https://github.com/BerriAI/litellm/pull/21212)
---
## AI Integrations
### Logging
- **[PostHog](../../docs/observability/posthog_integration)**
- Fix JSON serialization error for non-serializable objects - [PR #20668](https://github.com/BerriAI/litellm/pull/20668)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Sanitize label values to prevent metric scrape failures - [PR #20600](https://github.com/BerriAI/litellm/pull/20600)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Prevent empty proxy request spans from being sent to Langfuse - [PR #19935](https://github.com/BerriAI/litellm/pull/19935)
- **[OpenTelemetry](../../docs/proxy/logging#otel)**
- Auto-infer `otlp_http` exporter when endpoint is configured - [PR #20438](https://github.com/BerriAI/litellm/pull/20438)
- **[CloudZero](../../docs/proxy/logging)**
- Update CBF field mappings per LIT-1907 - [PR #20906](https://github.com/BerriAI/litellm/pull/20906)
- **General**
- Allow `MAX_CALLBACKS` override via env var - [PR #20781](https://github.com/BerriAI/litellm/pull/20781)
- Add `standard_logging_payload_excluded_fields` config option - [PR #20831](https://github.com/BerriAI/litellm/pull/20831)
- Enable `verbose_logger` when `LITELLM_LOG=DEBUG` - [PR #20496](https://github.com/BerriAI/litellm/pull/20496)
- Guard against None `litellm_metadata` in batch logging path - [PR #20832](https://github.com/BerriAI/litellm/pull/20832)
- Propagate model-level tags from config to SpendLogs - [PR #20769](https://github.com/BerriAI/litellm/pull/20769)
### Guardrails
- **Policy Templates**
- New Policy Templates: pre-configured guardrail combinations for specific use-cases - [PR #21025](https://github.com/BerriAI/litellm/pull/21025)
- Add NSFW policy template, toxic keywords in multiple languages, child safety content filter, JSON content viewer - [PR #21205](https://github.com/BerriAI/litellm/pull/21205)
- Add toxic/abusive content filter guardrails - [PR #20934](https://github.com/BerriAI/litellm/pull/20934)
- **Pipeline Execution**
- Add guardrail pipeline support for conditional sequential execution - [PR #21177](https://github.com/BerriAI/litellm/pull/21177)
- Agent Guardrails on streaming output - [PR #21206](https://github.com/BerriAI/litellm/pull/21206)
- Pipeline flow builder UI - [PR #21188](https://github.com/BerriAI/litellm/pull/21188)
- **[Zscaler AI Guard](../../docs/apply_guardrail)**
- Zscaler AI Guard bug fixes and support during post-call - [PR #20801](https://github.com/BerriAI/litellm/pull/20801)
- Zscaler AI Guard UI - [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
- **[ZGuard](../../docs/apply_guardrail)**
- Add team policy mapping for ZGuard - [PR #20608](https://github.com/BerriAI/litellm/pull/20608)
- **General**
- Add logging to all unified guardrails + link to custom code guardrail templates - [PR #20900](https://github.com/BerriAI/litellm/pull/20900)
- Forward request headers + `litellm_version` to generic guardrails - [PR #20729](https://github.com/BerriAI/litellm/pull/20729)
- Empty `guardrails`/`policies` arrays should not trigger enterprise license check - [PR #20567](https://github.com/BerriAI/litellm/pull/20567)
- Fix OpenAI moderation guardrails - [PR #20718](https://github.com/BerriAI/litellm/pull/20718)
- Fix `/v2/guardrails/list` returning sensitive values - [PR #20796](https://github.com/BerriAI/litellm/pull/20796)
- Fix guardrail status error - [PR #20972](https://github.com/BerriAI/litellm/pull/20972)
- Reuse `get_instance_fn` in `initialize_custom_guardrail` - [PR #20917](https://github.com/BerriAI/litellm/pull/20917)
---
## Spend Tracking, Budgets and Rate Limiting
- **Prevent shared backend model key from being polluted** by per-deployment custom pricing - [PR #20679](https://github.com/BerriAI/litellm/pull/20679)
- **Avoid in-place mutation** in SpendUpdateQueue aggregation - [PR #20876](https://github.com/BerriAI/litellm/pull/20876)
---
## MCP Gateway (12 updates)
- **MCP M2M OAuth2 Support** - Add support for machine-to-machine OAuth2 for MCP servers - [PR #20788](https://github.com/BerriAI/litellm/pull/20788)
- **MCP Server Discovery UI** - Browse and discover available MCP servers from the UI - [PR #21079](https://github.com/BerriAI/litellm/pull/21079)
- **MCP Tracing** - Add OpenTelemetry tracing for MCP calls running through AI Gateway - [PR #21018](https://github.com/BerriAI/litellm/pull/21018)
- **MCP OAuth2 Debug Headers** - Client-side debug headers for OAuth2 troubleshooting - [PR #21151](https://github.com/BerriAI/litellm/pull/21151)
- **Fix MCP "Session not found" errors** - Resolve session persistence issues - [PR #21040](https://github.com/BerriAI/litellm/pull/21040)
- **Fix MCP OAuth2 root endpoints** returning "MCP server not found" - [PR #20784](https://github.com/BerriAI/litellm/pull/20784)
- **Fix MCP OAuth2 query param merging** when `authorization_url` already contains params - [PR #20968](https://github.com/BerriAI/litellm/pull/20968)
- **Fix MCP SCOPES on Atlassian** issue - [PR #21150](https://github.com/BerriAI/litellm/pull/21150)
- **Fix MCP StreamableHTTP backend** - Use `anyio.fail_after` instead of `asyncio.wait_for` - [PR #20891](https://github.com/BerriAI/litellm/pull/20891)
- **Inject `NPM_CONFIG_CACHE`** into STDIO MCP subprocess env - [PR #21069](https://github.com/BerriAI/litellm/pull/21069)
- **Block spaces and hyphens** in MCP server names and aliases - [PR #21074](https://github.com/BerriAI/litellm/pull/21074)
---
## Performance / Loadbalancing / Reliability improvements (8 improvements)
- **Remove orphan entries from queue** - Fix memory leak in scheduler queue - [PR #20866](https://github.com/BerriAI/litellm/pull/20866)
- **Remove repeated provider parsing** in budget limiter hot path - [PR #21043](https://github.com/BerriAI/litellm/pull/21043)
- **Use current retry exception** for retry backoff instead of stale exception - [PR #20725](https://github.com/BerriAI/litellm/pull/20725)
- **Add Semgrep & fix OOMs** - Static analysis rules and out-of-memory fixes - [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
- **Add Pyroscope** for continuous profiling and observability - [PR #21167](https://github.com/BerriAI/litellm/pull/21167)
- **Respect `ssl_verify`** with shared aiohttp sessions - [PR #20349](https://github.com/BerriAI/litellm/pull/20349)
- **Fix shared health check serialization** - [PR #21119](https://github.com/BerriAI/litellm/pull/21119)
- **Change model mismatch logs** from WARNING to DEBUG - [PR #20994](https://github.com/BerriAI/litellm/pull/20994)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR | Migration |
| ----- | ----------- | ----------- | -- | --------- |
| `LiteLLM_VerificationToken` | New Indexes | Added indexes on `user_id`+`team_id`, `team_id`, and `budget_reset_at`+`expires` | [PR #20736](https://github.com/BerriAI/litellm/pull/20736) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql) |
| `LiteLLM_PolicyAttachmentTable` | New Column | Added `tags` text array for policy-to-tag connections | [PR #21061](https://github.com/BerriAI/litellm/pull/21061) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql) |
| `LiteLLM_AccessGroupTable` | New Table | Access groups for managing model, MCP server, and agent access | [PR #21022](https://github.com/BerriAI/litellm/pull/21022) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql) |
| `LiteLLM_AccessGroupTable` | Column Change | Renamed `access_model_ids` to `access_model_names` | [PR #21166](https://github.com/BerriAI/litellm/pull/21166) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213170952_access_group_change_to_model_name/migration.sql) |
| `LiteLLM_ManagedVectorStoreTable` | New Table | Managed vector store tracking with model mappings | - | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql) |
| `LiteLLM_TeamTable`, `LiteLLM_VerificationToken` | New Column | Added `access_group_ids` text array | [PR #21022](https://github.com/BerriAI/litellm/pull/21022) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql) |
| `LiteLLM_GuardrailsTable` | New Column | Added `team_id` text column | - | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql) |
---
## Documentation Updates (14 updates)
- LiteLLM Observatory section added to v1.81.9 release notes - [PR #20675](https://github.com/BerriAI/litellm/pull/20675)
- Callback registration optimization added to release notes - [PR #20681](https://github.com/BerriAI/litellm/pull/20681)
- Middleware performance blog post - [PR #20677](https://github.com/BerriAI/litellm/pull/20677)
- UI Team Soft Budget documentation - [PR #20669](https://github.com/BerriAI/litellm/pull/20669)
- UI Contributing and Troubleshooting guide - [PR #20674](https://github.com/BerriAI/litellm/pull/20674)
- Reorganize Admin UI subsection - [PR #20676](https://github.com/BerriAI/litellm/pull/20676)
- SDK proxy authentication (OAuth2/JWT auto-refresh) - [PR #20680](https://github.com/BerriAI/litellm/pull/20680)
- Forward client headers to LLM API documentation fix - [PR #20768](https://github.com/BerriAI/litellm/pull/20768)
- Add docs guide for using policies - [PR #20914](https://github.com/BerriAI/litellm/pull/20914)
- Add native thinking param examples for Claude Opus 4.6 - [PR #20799](https://github.com/BerriAI/litellm/pull/20799)
- Fix Claude Code MCP tutorial - [PR #21145](https://github.com/BerriAI/litellm/pull/21145)
- Add API base URLs for Dashscope (International and China/Beijing) - [PR #21083](https://github.com/BerriAI/litellm/pull/21083)
- Fix `DEFAULT_NUM_WORKERS_LITELLM_PROXY` default (1, not 4) - [PR #21127](https://github.com/BerriAI/litellm/pull/21127)
- Correct ElevenLabs support status in README - [PR #20643](https://github.com/BerriAI/litellm/pull/20643)
---
## New Contributors
* @iver56 made their first contribution in [PR #20643](https://github.com/BerriAI/litellm/pull/20643)
* @eliasaronson made their first contribution in [PR #20666](https://github.com/BerriAI/litellm/pull/20666)
* @NirantK made their first contribution in [PR #19656](https://github.com/BerriAI/litellm/pull/19656)
* @looksgood made their first contribution in [PR #20919](https://github.com/BerriAI/litellm/pull/20919)
* @kelvin-tran made their first contribution in [PR #20548](https://github.com/BerriAI/litellm/pull/20548)
* @bluet made their first contribution in [PR #20873](https://github.com/BerriAI/litellm/pull/20873)
* @itayov made their first contribution in [PR #20729](https://github.com/BerriAI/litellm/pull/20729)
* @CSteigstra made their first contribution in [PR #20960](https://github.com/BerriAI/litellm/pull/20960)
* @rahulrd25 made their first contribution in [PR #20569](https://github.com/BerriAI/litellm/pull/20569)
* @muraliavarma made their first contribution in [PR #20598](https://github.com/BerriAI/litellm/pull/20598)
* @joaokopernico made their first contribution in [PR #21039](https://github.com/BerriAI/litellm/pull/21039)
* @datzscaler made their first contribution in [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
* @atapia27 made their first contribution in [PR #20922](https://github.com/BerriAI/litellm/pull/20922)
* @fpagny made their first contribution in [PR #21121](https://github.com/BerriAI/litellm/pull/21121)
* @aidankovacic-8451 made their first contribution in [PR #21119](https://github.com/BerriAI/litellm/pull/21119)
* @luisgallego-aily made their first contribution in [PR #19935](https://github.com/BerriAI/litellm/pull/19935)
---
## Full Changelog
[v1.81.9.rc.1...v1.81.12.rc.1](https://github.com/BerriAI/litellm/compare/v1.81.9.rc.1...v1.81.12.rc.1)
docs/my-website/release_notes/v1.81.14.md
+447
View File
@@ -0,0 +1,447 @@
---
title: "v1.81.14-stable - Claude Sonnet 4.6, Guardrail Garden & Major Performance Improvements"
slug: "v1-81-14"
date: 2026-02-21T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.14-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.14
```
</TabItem>
</Tabs>
## Key Highlights
- **Use Claude Sonnet 4.6 on day 0** — [reasoning, computer use, prompt caching, and 200K context, working across Anthropic and Vertex AI from the moment it launched](../../docs/providers/anthropic)
- **Deploy guardrails without writing code** — [Guardrail Garden lets you browse a marketplace of pre-built policies (competitor blockers, GDPR PII, EU AI Act, prompt injection) and deploy in one click](../../docs/proxy/guardrails/policy_templates)
- **Test guardrail policies before shipping** — [upload a CSV dataset to the compliance playground and validate policies against real traffic; get AI-generated policy suggestions with latency overhead estimates](../../docs/proxy/guardrails/policy_templates)
- **Turn any OpenAPI spec into an MCP server** — [paste a spec and get a working MCP server instantly, via API or UI](../../docs/mcp)
- **Call any prompt management system from a single API** — [the new Prompt Management API works with Langfuse, LangSmith, and others without requiring per-integration code](../../docs/proxy/prompt_management)
- **Major performance batch** — 20+ targeted optimizations across router algorithms, logging overhead, cost calculator, and connection management — meaningfully lower latency and CPU overhead on every request
---
This release includes the largest single batch of performance work since v1.74. The most impactful change moves async/sync callback sorting from per-request to registration time (~30% speedup for callback-heavy deployments). On top of that: Pydantic round-trips eliminated from the logging hot path, OpenAI client init params pre-computed once at startup, quadratic deployment scan removed from usage-based routing, and several O(n²) → O(1) fixes in the router's team filter and model list lookups. Combined, these changes add up for high-throughput deployments that were hitting CPU ceilings.
---
## New Providers and Endpoints
### New Providers (1 new provider)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | --------------------------- | ----------- |
| [IBM watsonx.ai](../../docs/providers/watsonx) | `/rerank` | Rerank support for IBM watsonx.ai models |
### New LLM API Endpoints (1 new endpoint)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
| `/v1/evals` | POST/GET | OpenAI-compatible Evals API for model evaluation | [Docs](../../docs/evals_api) |
---
## New Models / Updated Models
#### New Model Support (13 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Anthropic | `claude-sonnet-4-6` | 200K | $3.00 | $15.00 | Reasoning, computer use, prompt caching, vision, PDF |
| Vertex AI | `vertex_ai/claude-opus-4-6@default` | 1M | $5.00 | $25.00 | Reasoning, computer use, prompt caching |
| Google Gemini | `gemini/gemini-3.1-pro-preview` | 1M | $2.00 | $12.00 | Audio, video, images, PDF |
| Google Gemini | `gemini/gemini-3.1-pro-preview-customtools` | 1M | $2.00 | $12.00 | Custom tools |
| GitHub Copilot | `github_copilot/gpt-5.3-codex` | 128K | - | - | Responses API, function calling, vision |
| GitHub Copilot | `github_copilot/claude-opus-4.6-fast` | 128K | - | - | Chat completions, function calling, vision |
| Mistral | `mistral/devstral-small-latest` | 256K | $0.10 | $0.30 | Function calling, response schema |
| Mistral | `mistral/devstral-latest` | 256K | $0.40 | $2.00 | Function calling, response schema |
| Mistral | `mistral/devstral-medium-latest` | 256K | $0.40 | $2.00 | Function calling, response schema |
| OpenRouter | `openrouter/minimax/minimax-m2.5` | 196K | $0.30 | $1.10 | Function calling, reasoning, prompt caching |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/glm-4p7` | - | - | - | Chat completions |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/minimax-m2p1` | - | - | - | Chat completions |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/kimi-k2p5` | - | - | - | Chat completions |
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Day 0 support for Claude Sonnet 4.6 with reasoning, computer use, and 200K context - [PR #21401](https://github.com/BerriAI/litellm/pull/21401)
- Add Claude Sonnet 4.6 pricing - [PR #21395](https://github.com/BerriAI/litellm/pull/21395)
- Add day 0 feature support for Claude Sonnet 4.6 (streaming, function calling, vision) - [PR #21448](https://github.com/BerriAI/litellm/pull/21448)
- Add `reasoning` effort and extended thinking support for Sonnet 4.6 - [PR #21598](https://github.com/BerriAI/litellm/pull/21598)
- Fix empty system messages in `translate_system_message` - [PR #21630](https://github.com/BerriAI/litellm/pull/21630)
- Sanitize Anthropic messages for multi-turn compatibility - [PR #21464](https://github.com/BerriAI/litellm/pull/21464)
- Map `websearch` tool from `/v1/messages` to `/chat/completions` - [PR #21465](https://github.com/BerriAI/litellm/pull/21465)
- Forward `reasoning` field as `reasoning_content` in delta streaming - [PR #21468](https://github.com/BerriAI/litellm/pull/21468)
- Add server-side compaction translation from OpenAI to Anthropic format - [PR #21555](https://github.com/BerriAI/litellm/pull/21555)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Native structured outputs API support (`outputConfig.textFormat`) - [PR #21222](https://github.com/BerriAI/litellm/pull/21222)
- Support `nova/` and `nova-2/` spec prefixes for custom imported models - [PR #21359](https://github.com/BerriAI/litellm/pull/21359)
- Broaden Nova 2 model detection to support all `nova-2-*` variants - [PR #21358](https://github.com/BerriAI/litellm/pull/21358)
- Add Accept header for AgentCore MCP server requests - [PR #21551](https://github.com/BerriAI/litellm/pull/21551)
- Clamp `thinking.budget_tokens` to minimum 1024 - [PR #21306](https://github.com/BerriAI/litellm/pull/21306)
- Fix `parallel_tool_calls` mapping for Bedrock Converse - [PR #21659](https://github.com/BerriAI/litellm/pull/21659)
- **[Google Gemini / Vertex AI](../../docs/providers/gemini)**
- Day 0 support for `gemini-3.1-pro-preview` - [PR #21568](https://github.com/BerriAI/litellm/pull/21568)
- Fix `_map_reasoning_effort_to_thinking_level` for all Gemini 3 family models - [PR #21654](https://github.com/BerriAI/litellm/pull/21654)
- Add reasoning support via config for Gemini models - [PR #21663](https://github.com/BerriAI/litellm/pull/21663)
- **[Databricks](../../docs/providers/databricks)**
- Add Databricks to supported providers for response schema - [PR #21368](https://github.com/BerriAI/litellm/pull/21368)
- Native Responses API support for Databricks GPT models - [PR #21460](https://github.com/BerriAI/litellm/pull/21460)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Add `github_copilot/gpt-5.3-codex` and `github_copilot/claude-opus-4.6-fast` models - [PR #21316](https://github.com/BerriAI/litellm/pull/21316)
- Fix unsupported params for ChatGPT Codex - [PR #21209](https://github.com/BerriAI/litellm/pull/21209)
- Allow GitHub model aliases to reuse upstream model metadata - [PR #21497](https://github.com/BerriAI/litellm/pull/21497)
- **[Mistral](../../docs/providers/mistral)**
- Add `devstral-2512` model aliases (`devstral-small-latest`, `devstral-latest`, `devstral-medium-latest`) - [PR #21372](https://github.com/BerriAI/litellm/pull/21372)
- **[IBM watsonx.ai](../../docs/providers/watsonx)**
- Add native rerank support - [PR #21303](https://github.com/BerriAI/litellm/pull/21303)
- **[xAI](../../docs/providers/xai)**
- Fix usage object in xAI responses - [PR #21559](https://github.com/BerriAI/litellm/pull/21559)
- **[Dashscope](../../docs/providers/dashscope)**
- Remove list-to-str transformation that caused incorrect request formatting - [PR #21547](https://github.com/BerriAI/litellm/pull/21547)
- **[hosted_vllm](../../docs/providers/vllm)**
- Convert thinking blocks to content blocks for multi-turn conversations - [PR #21557](https://github.com/BerriAI/litellm/pull/21557)
- **[OCI / Oracle](../../docs/providers/oci_cohere)**
- Fix Grok output pricing - [PR #21329](https://github.com/BerriAI/litellm/pull/21329)
- **[AU Anthropic](../../docs/providers/anthropic)**
- Fix `au.anthropic.claude-opus-4-6-v1` model ID - [PR #20731](https://github.com/BerriAI/litellm/pull/20731)
- **General**
- Add routing based on reasoning support — skip deployments that don't support reasoning when `thinking` params are present - [PR #21302](https://github.com/BerriAI/litellm/pull/21302)
- Add `stop` as supported param for OpenAI and Azure - [PR #21539](https://github.com/BerriAI/litellm/pull/21539)
- Add `store` and other missing params to `OPENAI_CHAT_COMPLETION_PARAMS` - [PR #21195](https://github.com/BerriAI/litellm/pull/21195), [PR #21360](https://github.com/BerriAI/litellm/pull/21360)
- Preserve `provider_specific_fields` from proxy responses - [PR #21220](https://github.com/BerriAI/litellm/pull/21220)
- Add default usage data configuration - [PR #21550](https://github.com/BerriAI/litellm/pull/21550)
### Bug Fixes
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Fix service_tier cost propagation - [PR #21172](https://github.com/BerriAI/litellm/pull/21172)
- Fix per-image pricing for multimodal embeddings - [PR #21646](https://github.com/BerriAI/litellm/pull/21646)
- Use `batch_` prefix for Vertex AI batch IDs in `encode_file_id_with_model` - [PR #21624](https://github.com/BerriAI/litellm/pull/21624)
- **[Bedrock Converse](../../docs/providers/bedrock)**
- Fix Anthropic usage object to match v1/messages spec - [PR #21295](https://github.com/BerriAI/litellm/pull/21295)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Add missing model pricing for `glm-4p7`, `minimax-m2p1`, `kimi-k2p5` - [PR #21642](https://github.com/BerriAI/litellm/pull/21642)
- **[Responses API](../../docs/response_api)**
- Fix `use None` instead of `Reasoning()` for reasoning parameter - [PR #21103](https://github.com/BerriAI/litellm/pull/21103)
- Preserve metadata for custom callbacks on codex/responses path - [PR #21243](https://github.com/BerriAI/litellm/pull/21243)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Return `finish_reason='tool_calls'` when response contains function_call items - [PR #19745](https://github.com/BerriAI/litellm/pull/19745)
- Eliminate per-chunk thread spawning in async streaming path for significantly better throughput - [PR #21709](https://github.com/BerriAI/litellm/pull/21709)
- **[Evals API](../../docs/evals_api)**
- Add support for OpenAI Evals API - [PR #21375](https://github.com/BerriAI/litellm/pull/21375)
- **[Batch API](../../docs/batches)**
- Add file deletion criteria with batch references - [PR #21456](https://github.com/BerriAI/litellm/pull/21456)
- Misc bug fixes for managed batches - [PR #21157](https://github.com/BerriAI/litellm/pull/21157)
- **[Pass-Through Endpoints](../../docs/pass_through/bedrock)**
- Add method-based routing for passthrough endpoints - [PR #21543](https://github.com/BerriAI/litellm/pull/21543)
- Preserve and forward OAuth Authorization headers through proxy layer - [PR #19912](https://github.com/BerriAI/litellm/pull/19912)
- **[Websearch / Tool Calling](../../docs/completion/input)**
- Add DuckDuckGo as a search tool - [PR #21467](https://github.com/BerriAI/litellm/pull/21467)
- Fix `pre_call_deployment_hook` not triggering via proxy router for websearch - [PR #21433](https://github.com/BerriAI/litellm/pull/21433)
- **General**
- Exclude tool params for models without function calling support - [PR #21244](https://github.com/BerriAI/litellm/pull/21244)
- Add `store` param to OpenAI chat completion params - [PR #21195](https://github.com/BerriAI/litellm/pull/21195)
- Add reasoning support via config for per-model reasoning configuration - [PR #21663](https://github.com/BerriAI/litellm/pull/21663)
#### Bugs
- **General**
- Fix `api_base` resolution error for models with multiple potential endpoints - [PR #21658](https://github.com/BerriAI/litellm/pull/21658)
- Fix session grouping broken for dict rows from `query_raw` - [PR #21435](https://github.com/BerriAI/litellm/pull/21435)
---
## Management Endpoints / UI
#### Features
- **Access Groups**
- Add Access Group Selector to Create and Edit flow for Keys/Teams - [PR #21234](https://github.com/BerriAI/litellm/pull/21234)
- **Virtual Keys**
- Fix virtual key grace period from env/UI - [PR #20321](https://github.com/BerriAI/litellm/pull/20321)
- Fix key expiry default duration - [PR #21362](https://github.com/BerriAI/litellm/pull/21362)
- Key Last Active Tracking — see when a key was last used - [PR #21545](https://github.com/BerriAI/litellm/pull/21545)
- Fix `/v1/models` returning wildcard instead of expanded models for BYOK team keys - [PR #21408](https://github.com/BerriAI/litellm/pull/21408)
- Return `failed_tokens` in delete_verification_tokens response - [PR #21609](https://github.com/BerriAI/litellm/pull/21609)
- **Models + Endpoints**
- Add Model Settings Modal to Models & Endpoints page - [PR #21516](https://github.com/BerriAI/litellm/pull/21516)
- Allow `store_model_in_db` to be set via database (not just config) - [PR #21511](https://github.com/BerriAI/litellm/pull/21511)
- Fix `input_cost_per_token` masked/hidden in Model Info UI - [PR #21723](https://github.com/BerriAI/litellm/pull/21723)
- Fix credentials for UI-created models in batch file uploads - [PR #21502](https://github.com/BerriAI/litellm/pull/21502)
- Resolve credentials for UI-created models - [PR #21502](https://github.com/BerriAI/litellm/pull/21502)
- **Teams**
- Allow team members to view entire team usage - [PR #21537](https://github.com/BerriAI/litellm/pull/21537)
- Fix service account visibility for team members - [PR #21627](https://github.com/BerriAI/litellm/pull/21627)
- Organization Info page: show member email, AntD tabs, reusable MemberTable - [PR #21745](https://github.com/BerriAI/litellm/pull/21745)
- **Usage / Spend Logs**
- Allow filtering Usage by User - [PR #21351](https://github.com/BerriAI/litellm/pull/21351)
- Inject Credential Name as Tag for Usage Page filtering - [PR #21715](https://github.com/BerriAI/litellm/pull/21715)
- Prefix credential tags and update Tag usage banner - [PR #21739](https://github.com/BerriAI/litellm/pull/21739)
- Show retry count for requests in Logs view - [PR #21704](https://github.com/BerriAI/litellm/pull/21704)
- Fix Aggregated Daily Activity Endpoint performance - [PR #21613](https://github.com/BerriAI/litellm/pull/21613)
- **SSO / Auth**
- Fix SSO PKCE support in multi-pod Kubernetes deployments - [PR #20314](https://github.com/BerriAI/litellm/pull/20314)
- Preserve SSO role regardless of `role_mappings` config - [PR #21503](https://github.com/BerriAI/litellm/pull/21503)
- **Proxy CLI / Master Key**
- Fix master key rotation Prisma validation errors - [PR #21330](https://github.com/BerriAI/litellm/pull/21330)
- Handle missing `DATABASE_URL` in `append_query_params` - [PR #21239](https://github.com/BerriAI/litellm/pull/21239)
- **Project Management**
- Add Project Management APIs for organizing resources - [PR #21078](https://github.com/BerriAI/litellm/pull/21078)
- **UI Improvements**
- Content Filters: help edit/view categories and 1-click add with pagination - [PR #21223](https://github.com/BerriAI/litellm/pull/21223)
- Playground: test fallbacks with UI - [PR #21007](https://github.com/BerriAI/litellm/pull/21007)
- Add `forward_client_headers_to_llm_api` toggle to general settings - [PR #21776](https://github.com/BerriAI/litellm/pull/21776)
- Fix `is_premium()` debug log spam on every request - [PR #20841](https://github.com/BerriAI/litellm/pull/20841)
#### Bugs
- Spend Logs: Fix cost calculation - [PR #21152](https://github.com/BerriAI/litellm/pull/21152)
- Logs: Fix table not updating and pagination issues - [PR #21708](https://github.com/BerriAI/litellm/pull/21708)
- Fix `/get_image` ignoring `UI_LOGO_PATH` when `cached_logo.jpg` exists - [PR #21637](https://github.com/BerriAI/litellm/pull/21637)
- Fix duplicate URL in `tagsSpendLogsCall` query string - [PR #20909](https://github.com/BerriAI/litellm/pull/20909)
- Preserve `key_alias` and `team_id` metadata in `/user/daily/activity/aggregated` after key deletion or regeneration - [PR #20684](https://github.com/BerriAI/litellm/pull/20684)
- Uncomment `response_model` in `user_info` endpoint - [PR #17430](https://github.com/BerriAI/litellm/pull/17430)
- Allow `internal_user_viewer` to access RAG endpoints; restrict ingest to existing vector stores - [PR #21508](https://github.com/BerriAI/litellm/pull/21508)
- Suppress warning for `litellm-dashboard` team in agent permission handler - [PR #21721](https://github.com/BerriAI/litellm/pull/21721)
---
## AI Integrations
### Logging
- **[DataDog](../../docs/proxy/logging#datadog)**
- Add `team` tag to logs, metrics, and cost management - [PR #21449](https://github.com/BerriAI/litellm/pull/21449)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Fix double-counting of `litellm_proxy_total_requests_metric` - [PR #21159](https://github.com/BerriAI/litellm/pull/21159)
- Guard against None metadata in Prometheus metrics - [PR #21489](https://github.com/BerriAI/litellm/pull/21489)
- Add ASGI middleware for improved Prometheus metrics collection - [PR #20434](https://github.com/BerriAI/litellm/pull/20434)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Improve Langfuse test isolation (multiple stability fixes) - [PR #21214](https://github.com/BerriAI/litellm/pull/21214)
- **General**
- Fix cost to 0 for cached responses in logging - [PR #21816](https://github.com/BerriAI/litellm/pull/21816)
- Improve streaming proxy throughput by fixing middleware and logging bottlenecks - [PR #21501](https://github.com/BerriAI/litellm/pull/21501)
- Reduce proxy overhead for large base64 payloads - [PR #21594](https://github.com/BerriAI/litellm/pull/21594)
- Close streaming connections to prevent connection pool exhaustion - [PR #21213](https://github.com/BerriAI/litellm/pull/21213)
### Guardrails
- **Guardrail Garden**
- Launch Guardrail Garden — a marketplace for pre-built guardrails deployable in one click - [PR #21732](https://github.com/BerriAI/litellm/pull/21732)
- Redesign guardrail creation form with vertical stepper UI - [PR #21727](https://github.com/BerriAI/litellm/pull/21727)
- Add guardrail jump link in log detail view - [PR #21437](https://github.com/BerriAI/litellm/pull/21437)
- Guardrail tracing UI: show policy, detection method, and match details - [PR #21349](https://github.com/BerriAI/litellm/pull/21349)
- **AI Policy Templates**
- Seven new ready-to-deploy policy templates ship in this release:
- GDPR Art. 32 EU PII Protection - [PR #21340](https://github.com/BerriAI/litellm/pull/21340)
- EU AI Act Article 5 (5 sub-guardrails, with French language support) - [PR #21342](https://github.com/BerriAI/litellm/pull/21342), [PR #21453](https://github.com/BerriAI/litellm/pull/21453), [PR #21427](https://github.com/BerriAI/litellm/pull/21427)
- Prompt injection detection - [PR #21520](https://github.com/BerriAI/litellm/pull/21520)
- Aviation and UAE topic filters with tag-based routing - [PR #21518](https://github.com/BerriAI/litellm/pull/21518)
- Airline off-topic restriction - [PR #21607](https://github.com/BerriAI/litellm/pull/21607)
- SQL injection - [PR #21806](https://github.com/BerriAI/litellm/pull/21806)
- AI-powered policy template suggestions with latency overhead estimates - [PR #21589](https://github.com/BerriAI/litellm/pull/21589), [PR #21608](https://github.com/BerriAI/litellm/pull/21608), [PR #21620](https://github.com/BerriAI/litellm/pull/21620)
- **Compliance Checker**
- Add compliance checker endpoints + UI panel - [PR #21432](https://github.com/BerriAI/litellm/pull/21432)
- CSV dataset upload to compliance playground for batch testing - [PR #21526](https://github.com/BerriAI/litellm/pull/21526)
- **Built-in Guardrails**
- Competitor name blocker: blocks by name, handles streaming, supports name variations, and splits pre/post call - [PR #21719](https://github.com/BerriAI/litellm/pull/21719), [PR #21533](https://github.com/BerriAI/litellm/pull/21533)
- Topic blocker with both keyword and embedding-based implementations - [PR #21713](https://github.com/BerriAI/litellm/pull/21713)
- Insults content filter - [PR #21729](https://github.com/BerriAI/litellm/pull/21729)
- MCP Security guardrail to block unregistered MCP servers - [PR #21429](https://github.com/BerriAI/litellm/pull/21429)
- **[Generic Guardrails](../../docs/proxy/guardrails)**
- Add configurable fallback to handle generic guardrail endpoint connection failures - [PR #21245](https://github.com/BerriAI/litellm/pull/21245)
- **[Presidio](../../docs/proxy/guardrails)**
- Fix Presidio controls configuration - [PR #21798](https://github.com/BerriAI/litellm/pull/21798)
- **[LakeraAI](../../docs/proxy/guardrails)**
- Avoid `KeyError` on missing `LAKERA_API_KEY` during initialization - [PR #21422](https://github.com/BerriAI/litellm/pull/21422)
### Prompt Management
- **Prompt Management API**
- New API to interact with prompt management integrations without requiring a PR - [PR #17800](https://github.com/BerriAI/litellm/pull/17800), [PR #17946](https://github.com/BerriAI/litellm/pull/17946)
- Fix prompt registry configuration issues - [PR #21402](https://github.com/BerriAI/litellm/pull/21402)
---
## Spend Tracking, Budgets and Rate Limiting
- **Fix Bedrock service_tier cost propagation** — costs from service-tier responses now correctly flow through to spend tracking - [PR #21172](https://github.com/BerriAI/litellm/pull/21172)
- **Fix cost for cached responses** — cached responses now correctly log $0 cost instead of re-billing - [PR #21816](https://github.com/BerriAI/litellm/pull/21816)
- **Aggregate daily activity endpoint performance** — faster queries for `/user/daily/activity/aggregated` - [PR #21613](https://github.com/BerriAI/litellm/pull/21613)
- **Preserve key_alias and team_id metadata** in `/user/daily/activity/aggregated` after key deletion or regeneration - [PR #20684](https://github.com/BerriAI/litellm/pull/20684)
- **Inject Credential Name as Tag** for granular usage page filtering by credential - [PR #21715](https://github.com/BerriAI/litellm/pull/21715)
---
## MCP Gateway
- **OpenAPI-to-MCP** — Convert any OpenAPI spec to an MCP server via API or UI - [PR #21575](https://github.com/BerriAI/litellm/pull/21575), [PR #21662](https://github.com/BerriAI/litellm/pull/21662)
- **MCP User Permissions** — Fine-grained permissions for end users on MCP servers - [PR #21462](https://github.com/BerriAI/litellm/pull/21462)
- **MCP Security Guardrail** — Block calls to unregistered MCP servers - [PR #21429](https://github.com/BerriAI/litellm/pull/21429)
- **Fix StreamableHTTPSessionManager** — Revert to stateless mode to prevent session state issues - [PR #21323](https://github.com/BerriAI/litellm/pull/21323)
- **Fix Bedrock AgentCore Accept header** — Add required Accept header for AgentCore MCP server requests - [PR #21551](https://github.com/BerriAI/litellm/pull/21551)
---
## Performance / Loadbalancing / Reliability improvements
**Logging & callback overhead**
- Move async/sync callback separation from per-request to callback registration time — ~30% speedup for callback-heavy deployments - [PR #20354](https://github.com/BerriAI/litellm/pull/20354)
- Skip Pydantic Usage round-trip in logging payload — reduces serialization overhead per request - [PR #21003](https://github.com/BerriAI/litellm/pull/21003)
- Skip duplicate `get_standard_logging_object_payload` calls for non-streaming requests - [PR #20440](https://github.com/BerriAI/litellm/pull/20440)
- Reuse `LiteLLM_Params` object across the request lifecycle - [PR #20593](https://github.com/BerriAI/litellm/pull/20593)
- Optimize `add_litellm_data_to_request` hot path - [PR #20526](https://github.com/BerriAI/litellm/pull/20526)
- Optimize `model_dump_with_preserved_fields` - [PR #20882](https://github.com/BerriAI/litellm/pull/20882)
- Pre-compute OpenAI client init params at module load instead of per-request - [PR #20789](https://github.com/BerriAI/litellm/pull/20789)
- Reduce proxy overhead for large base64 payloads - [PR #21594](https://github.com/BerriAI/litellm/pull/21594)
- Improve streaming proxy throughput by fixing middleware and logging bottlenecks - [PR #21501](https://github.com/BerriAI/litellm/pull/21501)
- Eliminate per-chunk thread spawning in Responses API async streaming - [PR #21709](https://github.com/BerriAI/litellm/pull/21709)
**Cost calculation**
- Optimize `completion_cost()` with early-exit and caching - [PR #20448](https://github.com/BerriAI/litellm/pull/20448)
- Cost calculator: reduce repeated lookups and dict copies - [PR #20541](https://github.com/BerriAI/litellm/pull/20541)
**Router & load balancing**
- Remove quadratic deployment scan in usage-based routing v2 - [PR #21211](https://github.com/BerriAI/litellm/pull/21211)
- Avoid O(n²) membership scans in team deployment filter - [PR #21210](https://github.com/BerriAI/litellm/pull/21210)
- Avoid O(n) alias scan for non-alias `get_model_list` lookups - [PR #21136](https://github.com/BerriAI/litellm/pull/21136)
- Increase default LRU cache size to reduce multi-model cache thrash - [PR #21139](https://github.com/BerriAI/litellm/pull/21139)
- Cache `get_model_access_groups()` no-args result on Router - [PR #20374](https://github.com/BerriAI/litellm/pull/20374)
- Deployment affinity routing callback — route to the same deployment for a session - [PR #19143](https://github.com/BerriAI/litellm/pull/19143)
- Complexity-based auto routing — new router strategy that routes based on request complexity - [PR #21789](https://github.com/BerriAI/litellm/pull/21789)
- Session-ID-based routing — use `session_id` for consistent routing within a session - [PR #21763](https://github.com/BerriAI/litellm/pull/21763)
**Connection management & reliability**
- Fix Redis connection pool reliability — prevent connection exhaustion under load - [PR #21717](https://github.com/BerriAI/litellm/pull/21717)
- Fix Prisma connection self-heal for auth and runtime reconnection (reverted, will be re-introduced with fixes) - [PR #21706](https://github.com/BerriAI/litellm/pull/21706)
- Make `PodLockManager.release_lock` atomic compare-and-delete - [PR #21226](https://github.com/BerriAI/litellm/pull/21226)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR |
| ----- | ----------- | ----------- | -- |
| `LiteLLM_DeletedVerificationToken` | New Column | Added `project_id` column | [PR #21587](https://github.com/BerriAI/litellm/pull/21587) |
| `LiteLLM_ProjectTable` | New Table | Project management for organizing resources | [PR #21078](https://github.com/BerriAI/litellm/pull/21078) |
| `LiteLLM_VerificationToken` | New Column | Added `last_active` timestamp for key activity tracking | [PR #21545](https://github.com/BerriAI/litellm/pull/21545) |
| `LiteLLM_ManagedVectorStoreTable` | Migration | Make vector store migration idempotent | [PR #21325](https://github.com/BerriAI/litellm/pull/21325) |
---
## Documentation Updates
- Add OpenAI Agents SDK with LiteLLM guide - [PR #21311](https://github.com/BerriAI/litellm/pull/21311)
- Access Groups documentation - [PR #21236](https://github.com/BerriAI/litellm/pull/21236)
- Anthropic beta headers documentation - [PR #21320](https://github.com/BerriAI/litellm/pull/21320)
- Latency overhead troubleshooting guide - [PR #21600](https://github.com/BerriAI/litellm/pull/21600), [PR #21603](https://github.com/BerriAI/litellm/pull/21603)
- Add rollback safety check guide - [PR #21743](https://github.com/BerriAI/litellm/pull/21743)
- Incident report: vLLM Embeddings broken by encoding_format parameter - [PR #21474](https://github.com/BerriAI/litellm/pull/21474)
- Incident report: Claude Code beta headers - [PR #21485](https://github.com/BerriAI/litellm/pull/21485)
- Mark v1.81.12 as stable - [PR #21809](https://github.com/BerriAI/litellm/pull/21809)
---
## New Contributors
* @mjkam made their first contribution in [PR #21306](https://github.com/BerriAI/litellm/pull/21306)
* @saneroen made their first contribution in [PR #21243](https://github.com/BerriAI/litellm/pull/21243)
* @vincentkoc made their first contribution in [PR #21239](https://github.com/BerriAI/litellm/pull/21239)
* @felixti made their first contribution in [PR #19745](https://github.com/BerriAI/litellm/pull/19745)
* @anttttti made their first contribution in [PR #20731](https://github.com/BerriAI/litellm/pull/20731)
* @ndgigliotti made their first contribution in [PR #21222](https://github.com/BerriAI/litellm/pull/21222)
* @iamadamreed made their first contribution in [PR #19912](https://github.com/BerriAI/litellm/pull/19912)
* @sahukanishka made their first contribution in [PR #21220](https://github.com/BerriAI/litellm/pull/21220)
* @namabile made their first contribution in [PR #21195](https://github.com/BerriAI/litellm/pull/21195)
* @stronk7 made their first contribution in [PR #21372](https://github.com/BerriAI/litellm/pull/21372)
* @ZeroAurora made their first contribution in [PR #21547](https://github.com/BerriAI/litellm/pull/21547)
* @SolitudePy made their first contribution in [PR #21497](https://github.com/BerriAI/litellm/pull/21497)
* @SherifWaly made their first contribution in [PR #21557](https://github.com/BerriAI/litellm/pull/21557)
* @dkindlund made their first contribution in [PR #21633](https://github.com/BerriAI/litellm/pull/21633)
* @cagojeiger made their first contribution in [PR #21664](https://github.com/BerriAI/litellm/pull/21664)
---
## Full Changelog
[v1.81.12.rc.1...v1.81.14.rc.1](https://github.com/BerriAI/litellm/compare/v1.81.12.rc.1...v1.81.14.rc.1)
docs/my-website/release_notes/v1.81.6.md
+8
View File
@@ -14,6 +14,14 @@ authors:
hide_table_of_contents: false
---
:::danger Known Issue - CPU Usage
This release had known issues with CPU usage. This has been fixed in [v1.81.9-stable](./v1-81-9).
**We recommend using v1.81.9-stable instead.**
:::
## Deploy this version
import Tabs from '@theme/Tabs';
docs/my-website/release_notes/v1.81.9.md
+12 -2
View File
@@ -1,5 +1,5 @@
---
title: "[Preview] v1.81.9 - Control which MCP Servers are exposed on the Internet"
title: "v1.81.9 - Control which MCP Servers are exposed on the Internet"
slug: "v1-81-9"
date: 2026-02-07T00:00:00
authors:
@@ -14,6 +14,16 @@ authors:
hide_table_of_contents: false
---
:::info Stable Release Branch
For each stable release, we now maintain a dedicated branch with the format `litellm_stable_release_branch_x_xx_xx` for the version.
This allows easier patching for day 0 model launches.
**Branch for v1.81.9:** [litellm_stable_release_branch_1_81_9](https://github.com/BerriAI/litellm/tree/litellm_stable_release_branch_1_81_9)
:::
## Deploy this version
import Tabs from '@theme/Tabs';
@@ -27,7 +37,7 @@ import Image from '@theme/IdealImage';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9.rc.1
ghcr.io/berriai/litellm:main-v1.81.9-stable
```
</TabItem>
docs/my-website/sidebars.js
+46 -15
View File
@@ -97,6 +97,7 @@ const sidebars = {
label: "Policies",
items: [
"proxy/guardrails/guardrail_policies",
"proxy/guardrails/policy_templates",
"proxy/guardrails/policy_tags",
],
},
@@ -106,7 +107,8 @@ const sidebars = {
items: [
"proxy/alerting",
"proxy/pagerduty",
"proxy/prometheus"
"proxy/prometheus",
"proxy/pyroscope_profiling"
]
},
{
@@ -118,6 +120,13 @@ const sidebars = {
type: "category",
label: "[Beta] Prompt Management",
items: [
{
type: "category",
label: "Contributing to Prompt Management",
items: [
"adding_provider/generic_prompt_management_api",
]
},
"proxy/litellm_prompt_management",
"proxy/custom_prompt_management",
"proxy/native_litellm_prompt",
@@ -174,6 +183,7 @@ const sidebars = {
"tutorials/copilotkit_sdk",
"tutorials/google_adk",
"tutorials/livekit_xai_realtime",
"projects/openai-agents"
]
},
@@ -400,6 +410,7 @@ const sidebars = {
items: [
"proxy/users",
"proxy/team_budgets",
"proxy/project_management",
"proxy/ui_team_soft_budget_alerts",
"proxy/tag_budgets",
"proxy/customers",
@@ -465,6 +476,7 @@ const sidebars = {
"proxy/model_access_guide",
"proxy/model_access",
"proxy/model_access_groups",
"proxy/access_groups",
"proxy/team_model_add"
]
},
@@ -569,6 +581,7 @@ const sidebars = {
"proxy/managed_finetuning",
]
},
"evals_api",
"generateContent",
"apply_guardrail",
"bedrock_invoke",
@@ -769,13 +782,13 @@ const sidebars = {
"providers/bedrock_batches",
"providers/bedrock_realtime_with_audio",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
@@ -873,6 +886,7 @@ const sidebars = {
},
"providers/sambanova",
"providers/sap",
"providers/scaleway",
"providers/stability",
"providers/synthetic",
"providers/snowflake",
@@ -931,6 +945,7 @@ const sidebars = {
"providers/anthropic_tool_search",
"guides/code_interpreter",
"completion/message_trimming",
"completion/message_sanitization",
"completion/model_alias",
"completion/mock_requests",
"completion/predict_outputs",
@@ -1002,6 +1017,7 @@ const sidebars = {
"tutorials/presidio_pii_masking",
"tutorials/elasticsearch_logging",
"tutorials/gemini_realtime_with_audio",
"tutorials/claude_code_beta_headers",
{
type: "category",
label: "LiteLLM Python SDK Tutorials",
@@ -1096,22 +1112,37 @@ const sidebars = {
"proxy_server",
],
},
"troubleshoot",
{
type: "category",
label: "Issue Reporting",
label: "Troubleshooting",
items: [
"troubleshoot/prisma_migrations",
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
"troubleshoot/ui_issues",
"mcp_troubleshoot",
{
type: "category",
label: "Performance / Latency",
items: [
"troubleshoot/latency_overhead",
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
"troubleshoot/prisma_migrations",
],
},
"troubleshoot/rollback",
"troubleshoot",
],
},
{
type: "category",
label: "Blog",
items: [
{
type: "link",
label: "Day 0 Support: Claude Sonnet 4.6",
href: "/blog/claude_sonnet_4_6",
},
{
type: "link",
label: "Incident: Broken Model Cost Map",
docs/my-website/src/pages/troubleshoot.md
-11
View File
@@ -1,11 +0,0 @@
# Troubleshooting
## Stable Version
If you're running into problems with installation / Usage
Use the stable version of litellm
```
pip install litellm==0.1.345
```
docs/my-website/static/img/project_spend.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 850 KiB

enterprise/dist/litellm_enterprise-0.1.32-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
enterprise/dist/litellm_enterprise-0.1.32.tar.gz
Vendored
BIN
View File
Binary file not shown.
enterprise/enterprise_hooks/__init__.py
+5
View File
@@ -1,11 +1,15 @@
from typing import Dict, Literal, Type, Union
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
from litellm_enterprise.proxy.hooks.managed_vector_stores import (
_PROXY_LiteLLMManagedVectorStores,
)
from litellm.integrations.custom_logger import CustomLogger
ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = {
"managed_files": _PROXY_LiteLLMManagedFiles,
"managed_vector_stores": _PROXY_LiteLLMManagedVectorStores,
}
@@ -13,6 +17,7 @@ def get_enterprise_proxy_hook(
hook_name: Union[
Literal[
"managed_files",
"managed_vector_stores",
"max_parallel_requests",
],
str,
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
+311 -309
View File
@@ -1,309 +1,311 @@
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import List, Literal, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get(
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: List[PagerDutyInternalEvent] = []
self._hanging_events: List[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
)
if not standard_logging_payload:
raise ValueError(
"standard_logging_object is required for PagerDutyAlerting"
)
# Extract error details
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information") or {}
)
_meta = standard_logging_payload.get("metadata") or {}
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"failure_threshold_window_seconds", 60
)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(
self.hanging_response_handler(
request_data=data, user_api_key_dict=user_api_key_dict
)
)
return None
async def hanging_response_handler(
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: List[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = (
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
)
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
) -> List[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get(
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: List[PagerDutyInternalEvent] = []
self._hanging_events: List[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
)
if not standard_logging_payload:
raise ValueError(
"standard_logging_object is required for PagerDutyAlerting"
)
# Extract error details
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information") or {}
)
_meta = standard_logging_payload.get("metadata") or {}
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"failure_threshold_window_seconds", 60
)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(
self.hanging_response_handler(
request_data=data, user_api_key_dict=user_api_key_dict
)
)
return None
async def hanging_response_handler(
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: List[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = (
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
)
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
) -> List[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
enterprise/litellm_enterprise/proxy/auth/route_checks.py
+5
View File
@@ -41,6 +41,10 @@ class EnterpriseRouteChecks:
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
@staticmethod
def should_call_route(route: str):
"""
@@ -58,6 +62,7 @@ class EnterpriseRouteChecks:
)
elif (
RouteChecks.is_llm_api_route(route=route)
and route not in EnterpriseRouteChecks.LLM_API_EXEMPT_ROUTES
and EnterpriseRouteChecks.is_llm_api_route_disabled()
):
raise HTTPException(
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+31 -20
View File
@@ -4,7 +4,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
from litellm._uuid import uuid
from datetime import datetime
from typing import TYPE_CHECKING, Optional, cast
from typing import TYPE_CHECKING, Optional
from litellm._logging import verbose_proxy_logger
@@ -35,14 +35,11 @@ class CheckBatchCost:
- if not, return False
- if so, return True
"""
from litellm_enterprise.proxy.hooks.managed_files import (
_PROXY_LiteLLMManagedFiles,
)
from litellm.batches.batch_utils import (
_get_file_content_as_dictionary,
calculate_batch_cost_and_usage,
)
from litellm.files.main import afile_content
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.openai_files_endpoints.common_utils import (
@@ -102,31 +99,41 @@ class CheckBatchCost:
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
managed_files_obj = cast(
Optional[_PROXY_LiteLLMManagedFiles],
self.proxy_logging_obj.get_proxy_hook("managed_files"),
)
if (
response.status == "completed"
and response.output_file_id is not None
and managed_files_obj is not None
):
verbose_proxy_logger.info(
f"Batch ID: {batch_id} is complete, tracking cost and usage"
)
# track cost
model_file_id_mapping = {
response.output_file_id: {model_id: response.output_file_id}
}
_file_content = await managed_files_obj.afile_content(
file_id=response.output_file_id,
litellm_parent_otel_span=None,
llm_router=self.llm_router,
model_file_id_mapping=model_file_id_mapping,
# This background job runs as default_user_id, so going through the HTTP endpoint
# would trigger check_managed_file_id_access and get 403. Instead, extract the raw
# provider file ID and call afile_content directly with deployment credentials.
raw_output_file_id = response.output_file_id
decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
if decoded:
try:
raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
except (IndexError, AttributeError):
pass
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
_file_content = await afile_content(
file_id=raw_output_file_id,
**credentials,
)
# Access content - handle both direct attribute and method call
if hasattr(_file_content, 'content'):
content_bytes = _file_content.content
elif hasattr(_file_content, 'read'):
content_bytes = await _file_content.read()
else:
content_bytes = _file_content
file_content_as_dict = _get_file_content_as_dictionary(
_file_content.content
content_bytes
)
deployment_info = self.llm_router.get_deployment(model_id=model_id)
@@ -143,11 +150,15 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
batch_cost, batch_usage, batch_models = (
await calculate_batch_cost_and_usage(
file_content_dictionary=file_content_as_dict,
custom_llm_provider=llm_provider, # type: ignore
model_name=model_name,
model_info=deployment_model_info,
)
)
logging_obj = LiteLLMLogging(
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+194 -11
View File
@@ -230,12 +230,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if managed_file:
return managed_file.created_by == user_id
return False
raise HTTPException(
status_code=404,
detail=f"File not found: {unified_file_id}",
)
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
@@ -246,7 +248,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if managed_object:
return managed_object.created_by == user_id
return True # don't raise error if managed object is not found
raise HTTPException(
status_code=404,
detail=f"Object not found: {unified_object_id}",
)
async def list_user_batches(
self,
@@ -911,15 +916,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
setattr(response, file_attr, unified_file_id)
# Fetch the actual file object from the provider
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id
)
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, 'llm_router', None)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
file_object = await litellm.afile_retrieve(
file_id=original_file_id,
**_creds,
)
else:
file_object = await litellm.afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id,
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={original_file_id}"
)
@@ -1004,8 +1018,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
if stored_file_object and stored_file_object.file_object:
return stored_file_object.file_object
# Use model_copy to ensure the ID update persists (Pydantic v2 compatibility)
response = stored_file_object.file_object.model_copy(update={"id": file_id})
return response
# Case 3: Managed file exists in the database but not the file object (for. e.g the batch task might not have run)
# So we fetch the file object from the provider. We deliberately do not store the result to avoid interfering with batch cost tracking code.
@@ -1033,6 +1051,168 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""Handled in files_endpoints.py"""
return []
def _is_batch_polling_enabled(self) -> bool:
"""
Check if batch cost tracking is actually enabled and running.
Returns:
bool: True if batch cost tracking is active, False otherwise
"""
try:
# Import here to avoid circular dependencies
import litellm.proxy.proxy_server as proxy_server_module
# Check if the scheduler has the batch cost checking job registered
scheduler = getattr(proxy_server_module, 'scheduler', None)
if scheduler is None:
return False
# Check if the check_batch_cost_job exists in the scheduler
try:
job = scheduler.get_job('check_batch_cost_job')
if job is not None:
return True
except Exception:
# Job not found or scheduler doesn't support get_job
pass
return False
except Exception as e:
verbose_logger.warning(
f"Error checking batch polling configuration: {e}. Assuming disabled."
)
return False
async def _get_batches_referencing_file(
self, file_id: str
) -> List[Dict[str, Any]]:
"""
Find batches in non-terminal states that reference this file.
Non-terminal states: validating, in_progress, finalizing
Terminal states: completed, complete, failed, expired, cancelled
Args:
file_id: The unified file ID to check
Returns:
List of batch objects referencing this file in non-terminal state
(max 10 for error message display)
"""
# Prepare list of file IDs to check (both unified and provider IDs)
file_ids_to_check = [file_id]
# Get model-specific file IDs for this unified file ID if it's a managed file
try:
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span=None
)
if model_file_id_mapping and file_id in model_file_id_mapping:
# Add all provider file IDs for this unified file
provider_file_ids = list(model_file_id_mapping[file_id].values())
file_ids_to_check.extend(provider_file_ids)
except Exception as e:
verbose_logger.debug(
f"Could not get model file ID mapping for {file_id}: {e}. "
f"Will only check unified file ID."
)
MAX_MATCHES_TO_RETURN = 10
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"file_purpose": "batch",
"status": {"in": ["validating", "in_progress", "finalizing"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
referencing_batches = []
for batch in batches:
try:
# Parse the batch file_object to check for file references
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")
referenced_file_ids = [fid for fid in [input_file_id, output_file_id, error_file_id] if fid]
# Check if any referenced file ID matches the file we're trying to delete
if any(ref_id in file_ids_to_check for ref_id in referenced_file_ids):
referencing_batches.append({
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
})
except Exception as e:
verbose_logger.warning(
f"Error parsing batch object {batch.unified_object_id}: {e}"
)
continue
return referencing_batches
async def _check_file_deletion_allowed(self, file_id: str) -> None:
"""
Check if file deletion should be blocked due to batch references.
Blocks deletion if:
1. File is referenced by any batch in non-terminal state, AND
2. Batch polling is configured (user wants cost tracking)
Args:
file_id: The unified file ID to check
Raises:
HTTPException: If file deletion should be blocked
"""
# Check if batch polling is enabled
if not self._is_batch_polling_enabled():
# Batch polling not configured, allow deletion
return
# Check if file is referenced by any non-terminal batches
referencing_batches = await self._get_batches_referencing_file(file_id)
if referencing_batches:
# File is referenced by non-terminal batches and polling is enabled
MAX_BATCHES_IN_ERROR = 5 # Limit batches shown in error message for readability
# Show up to MAX_BATCHES_IN_ERROR in the error message
batches_to_show = referencing_batches[:MAX_BATCHES_IN_ERROR]
batch_statuses = [f"{b['batch_id']}: {b['status']}" for b in batches_to_show]
# Determine the count message
count_message = f"{len(referencing_batches)}"
if len(referencing_batches) >= 10: # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
count_message = "10+"
error_message = (
f"Cannot delete file {file_id}. "
f"The file is referenced by {count_message} batch(es) in non-terminal state"
)
# Add specific batch details if not too many
if len(referencing_batches) <= MAX_BATCHES_IN_ERROR:
error_message += f": {', '.join(batch_statuses)}. "
else:
error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
error_message += (
f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
f"Alternatively, wait for all batches to complete processing."
)
raise HTTPException(
status_code=400,
detail=error_message,
)
async def afile_delete(
self,
file_id: str,
@@ -1041,6 +1221,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
**data: Dict,
) -> OpenAIFileObject:
# Check if file deletion should be blocked due to batch references
await self._check_file_deletion_allowed(file_id)
# file_id = convert_b64_uid_to_unified_uid(file_id)
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
+464
View File
@@ -0,0 +1,464 @@
# What is this?
## This hook is used to manage vector stores with target_model_names support
## It allows creating vector stores across multiple models and managing them with unified IDs
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from fastapi import HTTPException
import litellm
from litellm import Router, verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.managed_resources import BaseManagedResource
from litellm.llms.base_llm.managed_resources.utils import (
generate_unified_id_string,
is_base64_encoded_unified_id,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.vector_stores import (
VectorStoreCreateOptionalRequestParams,
VectorStoreCreateResponse,
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
Span = Any
InternalUsageCache = Any
PrismaClient = Any
class _PROXY_LiteLLMManagedVectorStores(
CustomLogger, BaseManagedResource[VectorStoreCreateResponse]
):
"""
Managed vector stores with target_model_names support.
This class provides functionality to:
- Create vector stores across multiple models
- Retrieve vector stores by unified ID
- Delete vector stores from all models
- List vector stores created by a user
"""
def __init__(
self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient
):
CustomLogger.__init__(self)
BaseManagedResource.__init__(self, internal_usage_cache, prisma_client)
# ============================================================================
# ABSTRACT METHOD IMPLEMENTATIONS
# ============================================================================
@property
def resource_type(self) -> str:
"""Return the resource type identifier."""
return "vector_store"
@property
def table_name(self) -> str:
"""Return the database table name for vector stores."""
# Prisma converts model name LiteLLM_ManagedVectorStoreTable to litellm_managedvectorstoretable
return "litellm_managedvectorstoretable"
def get_unified_resource_id_format(
self,
resource_object: VectorStoreCreateResponse,
target_model_names_list: List[str],
) -> str:
"""
Generate the format string for the unified vector store ID.
Format:
litellm_proxy:vector_store;unified_id,<uuid>;target_model_names,<models>;resource_id,<vs_id>;model_id,<model_id>
"""
# VectorStoreCreateResponse is a TypedDict, so resource_object is a dictionary
# Extract provider resource ID from the response
provider_resource_id = resource_object.get("id", "")
# Model ID is stored in hidden params if the response object supports it
# For TypedDict responses, we need to check if _hidden_params was added
hidden_params: Dict[str, Any] = {}
if hasattr(resource_object, "_hidden_params"):
hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", "")
return generate_unified_id_string(
resource_type=self.resource_type,
unified_uuid=str(uuid.uuid4()),
target_model_names=target_model_names_list,
provider_resource_id=provider_resource_id,
model_id=model_id,
)
async def create_resource_for_model(
self,
llm_router: Router,
model: str,
request_data: Dict[str, Any],
litellm_parent_otel_span: Span,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
Args:
llm_router: LiteLLM router instance
model: Model name to create vector store for
request_data: Request data for vector store creation
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
VectorStoreCreateResponse from the provider
"""
# Use the router to create the vector store
response = await llm_router.avector_store_create(
model=model, **request_data
)
return response
# ============================================================================
# VECTOR STORE CRUD OPERATIONS
# ============================================================================
async def acreate_vector_store(
self,
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""
Create a vector store across multiple models.
Args:
create_request: Vector store creation request parameters
llm_router: LiteLLM router instance
target_model_names_list: List of target model names
litellm_parent_otel_span: OpenTelemetry span for tracing
user_api_key_dict: User API key authentication details
Returns:
VectorStoreCreateResponse with unified ID
"""
verbose_logger.info(
f"Creating managed vector store for models: {target_model_names_list}"
)
# Create vector store for each model
# Convert TypedDict to Dict[str, Any] for base class compatibility
request_data_dict: Dict[str, Any] = dict(create_request)
responses = await self.create_resource_for_each_model(
llm_router=llm_router,
request_data=request_data_dict,
target_model_names_list=target_model_names_list,
litellm_parent_otel_span=litellm_parent_otel_span,
)
# Generate unified ID
unified_id = self.generate_unified_resource_id(
resource_objects=responses,
target_model_names_list=target_model_names_list,
)
# Extract model mappings from responses
model_mappings: Dict[str, str] = {}
for response in responses:
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id")
if model_id:
# VectorStoreCreateResponse is a TypedDict, use dict access
model_mappings[model_id] = response["id"]
verbose_logger.debug(
f"Created vector stores with model mappings: {model_mappings}"
)
# Store in database
await self.store_unified_resource_id(
unified_resource_id=unified_id,
resource_object=responses[0], # Store first response as template
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
# Return response with unified ID
# VectorStoreCreateResponse is a TypedDict, so we need to create a new dict with the unified ID
response = responses[0].copy()
response["id"] = unified_id
verbose_logger.info(
f"Successfully created managed vector store with unified ID: {unified_id}"
)
return response
async def alist_vector_stores(
self,
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
order: Optional[str] = None,
) -> Dict[str, Any]:
"""
List vector stores created by a user.
Args:
user_api_key_dict: User API key authentication details
limit: Maximum number of vector stores to return
after: Cursor for pagination
order: Sort order ('asc' or 'desc')
Returns:
Dictionary with list of vector stores and pagination info
"""
# Use the base class method
return await self.list_user_resources(
user_api_key_dict=user_api_key_dict,
limit=limit,
after=after,
)
# ============================================================================
# ACCESS CONTROL
# ============================================================================
async def check_vector_store_access(
self, vector_store_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
"""
Check if user has access to a vector store.
Args:
vector_store_id: The unified vector store ID
user_api_key_dict: User API key authentication details
Returns:
True if user has access, False otherwise
"""
is_unified_id = is_base64_encoded_unified_id(vector_store_id)
if is_unified_id:
# Check access for managed vector store
return await self.can_user_access_unified_resource_id(
vector_store_id,
user_api_key_dict,
)
# Not a managed vector store, allow access
return True
async def check_managed_vector_store_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth
) -> bool:
"""
Check if user has access to a managed vector store in request data.
Args:
data: Request data containing vector_store_id
user_api_key_dict: User API key authentication details
Returns:
True if this is a managed vector store and user has access
Raises:
HTTPException: If user doesn't have access
"""
vector_store_id = cast(Optional[str], data.get("vector_store_id"))
is_unified_id = (
is_base64_encoded_unified_id(vector_store_id)
if vector_store_id
else False
)
if is_unified_id and vector_store_id:
if await self.can_user_access_unified_resource_id(
vector_store_id, user_api_key_dict
):
return True
else:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}",
)
return False
# ============================================================================
# PRE-CALL HOOK (For Router Integration)
# ============================================================================
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: Any,
data: Dict,
call_type: str,
) -> Union[Exception, str, Dict, None]:
"""
Pre-call hook to handle vector store operations.
This hook intercepts vector store requests and:
- Validates access for managed vector stores
- Transforms unified IDs to provider-specific IDs
- Adds model routing information
Args:
user_api_key_dict: User API key authentication details
cache: Cache instance
data: Request data
call_type: Type of call being made
Returns:
Modified request data or None
"""
from litellm.llms.base_llm.managed_resources.utils import (
is_base64_encoded_unified_id,
parse_unified_id,
)
# Handle vector store search operations
if call_type == "avector_store_search":
vector_store_id = data.get("vector_store_id")
if vector_store_id:
# Check if it's a managed vector store ID
decoded_id = is_base64_encoded_unified_id(vector_store_id)
if decoded_id:
verbose_logger.debug(
f"Processing managed vector store search: {vector_store_id}"
)
# Check access
has_access = await self.can_user_access_unified_resource_id(
vector_store_id, user_api_key_dict
)
if not has_access:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}",
)
# Parse the unified ID to extract components
parsed_id = parse_unified_id(vector_store_id)
if parsed_id:
# Extract the model ID and provider resource ID
model_id = parsed_id.get("model_id")
provider_resource_id = parsed_id.get("provider_resource_id")
target_model_names = parsed_id.get("target_model_names", [])
verbose_logger.debug(
f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}"
)
# Determine which model to use for routing
# Priority: model_id (deployment ID) > first target_model_name
routing_model = None
if model_id:
routing_model = model_id
elif target_model_names and len(target_model_names) > 0:
routing_model = target_model_names[0]
# Set the model for routing
if routing_model:
data["model"] = routing_model
verbose_logger.info(
f"Routing vector store search to model: {routing_model}"
)
# Replace the unified ID with the provider-specific ID
if provider_resource_id:
data["vector_store_id"] = provider_resource_id
verbose_logger.debug(
f"Replaced unified ID with provider resource ID: {provider_resource_id}"
)
# Handle vector store retrieve/delete operations
elif call_type in ("avector_store_retrieve", "avector_store_delete"):
await self.check_managed_vector_store_access(data, user_api_key_dict)
# If it's a managed vector store, we'll handle it in the endpoint
# No need to transform here as the endpoint will route to the hook
return data
# ============================================================================
# POST-CALL HOOK (For Response Transformation)
# ============================================================================
async def async_post_call_success_hook(
self,
data: Dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
) -> Any:
"""
Post-call hook to transform responses.
This hook can be used to transform responses if needed.
For now, it just passes through the response.
Args:
data: Request data
user_api_key_dict: User API key authentication details
response: Response from the provider
Returns:
Potentially modified response
"""
# Currently no transformation needed
return response
# ============================================================================
# DEPLOYMENT FILTERING
# ============================================================================
async def async_filter_deployments( # type: ignore[override]
self,
model: str,
healthy_deployments: List,
messages: Optional[List] = None,
request_kwargs: Optional[Dict] = None,
parent_otel_span: Optional[Span] = None,
) -> List[Dict]:
"""
Filter deployments based on vector store availability.
This is used by the router to select only deployments that have
the vector store available.
Note: This method signature is a compromise between CustomLogger and BaseManagedResource
parent classes which have incompatible signatures. The type: ignore[override] is necessary
due to this multiple inheritance conflict.
Args:
model: Model name
healthy_deployments: List of healthy deployments
messages: Messages (unused for vector stores, required by CustomLogger interface)
request_kwargs: Request kwargs containing vector_store_id and mappings
parent_otel_span: OpenTelemetry span for tracing
Returns:
Filtered list of deployments
"""
return await BaseManagedResource.async_filter_deployments(
self,
model=model,
healthy_deployments=healthy_deployments,
request_kwargs=request_kwargs,
parent_otel_span=parent_otel_span,
resource_id_key="vector_store_id",
)
enterprise/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.31"
version = "0.1.32"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.31"
version = "0.1.32"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",
litellm-proxy-extras/build_and_publish.md
+127
View File
@@ -0,0 +1,127 @@
# Build & Publish `litellm-proxy-extras`
This runbook covers building and publishing a new version of the `litellm-proxy-extras` PyPI package. For use by litellm engineers only.
## Prerequisites
- All `schema.prisma` files are in sync (see [migration_runbook.md](./migration_runbook.md) Step 0)
- Migration has been generated and committed
- You are in the `litellm-proxy-extras/` directory
## Step 1: Bump the Version
### Option A: Automatic Version Bump (Recommended)
Use commitizen to automatically bump the version across all files:
```bash
cd litellm-proxy-extras
cz bump --increment patch
```
This will automatically:
- Bump the version in `pyproject.toml` (both `[tool.poetry].version` and `[tool.commitizen].version`)
- Update the version in `../requirements.txt`
- Update the version in `../pyproject.toml` (root)
- Create a git commit with the version bump
Then skip to Step 3 (Install Build Dependencies).
### Option B: Manual Version Bump
Update the version in `pyproject.toml`:
```bash
cd litellm-proxy-extras
# Check current version
grep 'version' pyproject.toml
```
Edit `pyproject.toml` and bump the version (both `[tool.poetry].version` and `[tool.commitizen].version`).
#### Step 2: Update Version in Root Package Files (Manual Only)
After bumping the version in `litellm-proxy-extras/pyproject.toml`, you **must** also update the version reference in the root-level files:
| File | Line to update |
|------|---------------|
| `requirements.txt` | `litellm-proxy-extras==X.Y.Z` |
| `pyproject.toml` (root) | `litellm-proxy-extras = {version = "X.Y.Z", optional = true}` |
```bash
# From the repo root — replace OLD with NEW version
sed -i '' 's/litellm-proxy-extras==OLD/litellm-proxy-extras==NEW/' requirements.txt
sed -i '' 's/litellm-proxy-extras = {version = "OLD"/litellm-proxy-extras = {version = "NEW"/' pyproject.toml
```
> **Do NOT skip this step.** The main `litellm` package pins the extras version — if you don't update these, users will install the old version.
## Step 3: Install Build Dependencies
```bash
pip install build twine
```
## Step 4: Clean Old Artifacts
```bash
rm -rf dist/ build/ *.egg-info
```
## Step 5: Build the Package
```bash
python3 -m build
```
This creates `.tar.gz` and `.whl` files in the `dist/` directory.
Verify the build output:
```bash
ls -la dist/
```
## Step 6: Upload to PyPI
```bash
twine upload dist/*
```
You will be prompted for your PyPI API token:
```
Enter your API token: pypi-...
```
> Use `__token__` as the username and your PyPI API token as the password.
## Quick Reference (Copy-Paste)
```bash
cd litellm-proxy-extras
rm -rf dist/ build/ *.egg-info
python3 -m build
twine upload dist/*
```
---
## Do you want to build and publish a new `litellm-proxy-extras` package? (y/n)
If **yes**, run the following commands in order:
```bash
cd litellm-proxy-extras
pip install build twine
rm -rf dist/ build/ *.egg-info
python3 -m build
twine upload dist/*
```
When `twine upload` runs, enter your PyPI credentials:
- **Username:** `__token__`
- **Password:** *(paste your PyPI API key)*
If **no**, you're done — no package publish needed.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.41-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.41.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.42-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.42.tar.gz
Vendored
BIN
View File
Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More