tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-06 04:23:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge: resolve conflicts with upstream/main

Browse Source 
- anthropic.md: keep claude-opus-4-6 alias and claude-sonnet-4-6 entry
- transformation.py: take upstream's formatted effort_map with fallback
This commit is contained in:
Chesars
2026-03-02 18:49:24 -03:00
parent c6b6e29bc2 835a2c3dc6
commit 6292c3dbdf
971 changed files with 38455 additions and 8799 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+71 -3
View File
@@ -1181,7 +1181,7 @@ jobs:
command: |
pwd
ls
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 4 --timeout=300 -vv --log-cli-level=INFO
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1699,7 +1699,7 @@ jobs:
command: |
prisma generate
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
@@ -3886,7 +3886,7 @@ jobs:
command: |
cd ~/project
# Check pyproject.toml
CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras'].split('\"')[1])")
CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
exit 1
@@ -4100,6 +4100,63 @@ jobs:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Sync schema on base e2e database
command: |
BASE_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch br-fancy-paper-ad1olsb3 \
--database-name yuneng-trial-db \
--role neondb_owner)
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$BASE_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
--name schema-sync \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--use_prisma_db_push
- run:
name: Start outputting logs
command: docker logs -f schema-sync
background: true
- run:
name: Wait for proxy to be ready (schema sync complete)
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Stop schema sync container
command: docker stop schema-sync
test_nonroot_image:
machine:
image: ubuntu-2204:2023.10.1
@@ -4298,6 +4355,15 @@ workflows:
only:
- main
- /litellm_.*/
- prisma_schema_sync:
context: e2e_ui_tests
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_chromium
browser: chromium
@@ -4305,6 +4371,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
@@ -4317,6 +4384,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
.github/codeql/codeql-config.yml
+15
View File
@@ -0,0 +1,15 @@
name: "LiteLLM CodeQL config"
# Exclude queries that produce result sets > 2 GiB on this codebase,
# causing 49+ minute runs that fail and block CI resources.
query-filters:
- exclude:
id: py/clear-text-logging-sensitive-data # CWE-312/CleartextLogging.ql — result set > 2 GiB
- exclude:
id: py/polynomial-redos # CWE-730/PolynomialReDoS.ql — result set > 2 GiB
paths-ignore:
- tests
- docs
- "**/*.md"
- litellm/proxy/_experimental/out
.github/observatory/litellm_config.yaml
+19
View File
@@ -0,0 +1,19 @@
# LiteLLM Observatory Test Configuration
# This config is used by CI to spin up a temporary LiteLLM instance
# for running observatory tests against RC/stable releases.
#
# Add model definitions for the providers you want to test.
# Provider API keys are injected via environment variables in CI.
model_list:
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
- model_name: gpt-4o-mini
litellm_params:
model: azure/gpt-4o-mini
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
.github/scripts/close_duplicate_issues.py
Executable
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""
Detect and close duplicate GitHub issues using title similarity.
Modes:
--scan Compare all open issues against each other (batch)
--issue-number N Check a single issue against older open issues
Requires the `gh` CLI to be authenticated.
"""
import argparse
import difflib
import json
import re
import subprocess
import sys
def normalize_title(title: str) -> str:
"""Strip common prefixes, lowercase, and collapse whitespace."""
title = re.sub(
r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
"",
title,
flags=re.IGNORECASE,
)
return " ".join(title.lower().split())
def gh(*args: str) -> str:
"""Run a gh CLI command and return stdout."""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def fetch_open_issues(repo: str | None) -> list[dict]:
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
if repo:
endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
else:
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
cmd = ["api", "--paginate", endpoint]
raw = gh(*cmd)
# gh --paginate concatenates JSON arrays, so we may get multiple arrays
issues = []
for line in raw.strip().splitlines():
line = line.strip()
if not line:
continue
parsed = json.loads(line)
if isinstance(parsed, list):
issues.extend(parsed)
else:
issues.append(parsed)
# Filter out pull requests (they also appear in the issues endpoint)
return [i for i in issues if "pull_request" not in i]
def close_as_duplicate(
issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
) -> None:
"""Close an issue as duplicate of another, adding a comment and label."""
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
return
# Add comment
comment_body = (
f"Closing as duplicate of #{duplicate_of}.\n\n"
"If you believe this is not a duplicate, please reopen and add context "
"explaining how this differs."
)
gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
# Add label
gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
# Close with not_planned reason
gh(
"api",
f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
"-X",
"PATCH",
"-f",
"state=closed",
"-f",
"state_reason=not_planned",
)
print(f" Closed #{issue_number} as duplicate of #{duplicate_of}")
def find_duplicate(
issue: dict, candidates: list[dict], threshold: float
) -> dict | None:
"""Return the first candidate whose normalized title is above threshold."""
norm = normalize_title(issue["title"])
for candidate in candidates:
if candidate["number"] == issue["number"]:
continue
cand_norm = normalize_title(candidate["title"])
ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
if ratio >= threshold:
return candidate
return None
def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
"""Compare every issue against all older issues. Returns count of duplicates found."""
# Sort oldest first
issues.sort(key=lambda i: i["number"])
closed_count = 0
for idx, issue in enumerate(issues):
older = issues[:idx]
if not older:
continue
dup = find_duplicate(issue, older, threshold)
if dup:
ratio = difflib.SequenceMatcher(
None,
normalize_title(issue["title"]),
normalize_title(dup["title"]),
).ratio()
print(
f"#{issue['number']}: \"{issue['title']}\"\n"
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
f"({ratio:.0%} similar)"
)
close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
closed_count += 1
return closed_count
def check_single(
issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
) -> bool:
"""Check a single issue against all older open issues. Returns True if duplicate found."""
target = None
for i in issues:
if i["number"] == issue_number:
target = i
break
if target is None:
print(f"Issue #{issue_number} not found among open issues.")
return False
older = [i for i in issues if i["number"] < issue_number]
dup = find_duplicate(target, older, threshold)
if dup:
ratio = difflib.SequenceMatcher(
None,
normalize_title(target["title"]),
normalize_title(dup["title"]),
).ratio()
print(
f"#{target['number']}: \"{target['title']}\"\n"
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
f"({ratio:.0%} similar)"
)
close_as_duplicate(issue_number, dup["number"], repo, dry_run)
return True
print(f"#{issue_number}: no duplicate found above threshold {threshold}")
return False
def main() -> None:
parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
args = parser.parse_args()
dry_run = not args.close
if dry_run:
print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
print("Fetching open issues...")
issues = fetch_open_issues(args.repo)
print(f"Found {len(issues)} open issues.\n")
if args.scan:
count = scan_all(issues, args.threshold, args.repo, dry_run)
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
else:
found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error
if __name__ == "__main__":
main()
.github/workflows/auto_update_price_and_context_window.yml
+1
View File
@@ -7,6 +7,7 @@ on:
jobs:
auto_update_price_and_context_window:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
.github/workflows/check_duplicate_issues.yml
+23
View File
@@ -27,3 +27,26 @@ jobs:
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
- name: Checkout close script
if: github.event.action == 'opened'
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts
- name: Set up Python
if: github.event.action == 'opened'
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Auto-close if high-confidence duplicate
if: github.event.action == 'opened'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
python3 .github/scripts/close_duplicate_issues.py \
--issue-number ${{ github.event.issue.number }} \
--repo ${{ github.repository }} \
--threshold 0.85 \
--close
.github/workflows/codeql.yml
+54
View File
@@ -0,0 +1,54 @@
name: "CodeQL"
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
# Run weekly on Sundays at 04:00 UTC
- cron: "0 4 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
name: Analyze (${{ matrix.language }})
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
security-events: write
packages: read
actions: read
contents: read
strategy:
fail-fast: false
matrix:
include:
- language: actions
build-mode: none
- language: javascript-typescript
build-mode: none
- language: python
build-mode: none
- language: ruby
build-mode: none
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{ matrix.language }}"
.github/workflows/ghcr_deploy.yml
+9
View File
@@ -299,6 +299,15 @@ jobs:
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
run-observatory-tests:
if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
needs: [docker-hub-deploy]
uses: ./.github/workflows/run_observatory_tests.yml
with:
tag: ${{ github.event.inputs.tag }}
commit_hash: ${{ github.event.inputs.commit_hash }}
secrets: inherit
build-and-push-helm-chart:
if: github.event.inputs.release_type != 'dev'
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
.github/workflows/publish_enterprise.yml
+74
View File
@@ -0,0 +1,74 @@
name: Publish litellm-enterprise to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
defaults:
run:
working-directory: enterprise
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
run: |
pip install twine
twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*
.github/workflows/publish_proxy_extras.yml
+74
View File
@@ -0,0 +1,74 @@
name: Publish litellm-proxy-extras to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
defaults:
run:
working-directory: litellm-proxy-extras
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
pip install twine
twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*
.github/workflows/run_observatory_tests.yml
+225
View File
@@ -0,0 +1,225 @@
name: Run Observatory Tests
on:
workflow_dispatch:
inputs:
tag:
description: "Docker image tag to test (e.g. v1.61.0.rc1)"
required: true
type: string
commit_hash:
description: "Commit hash (defaults to HEAD of current branch)"
required: false
type: string
workflow_call:
inputs:
tag:
description: "Docker image tag to test"
required: true
type: string
commit_hash:
description: "Commit hash of the release"
required: true
type: string
permissions:
contents: read
env:
LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
jobs:
observatory-tests:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate tag input
env:
TAG: ${{ inputs.tag }}
run: |
if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
exit 1
fi
- name: Start LiteLLM container
env:
TAG: ${{ inputs.tag }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
run: |
docker run -d \
--name litellm-rc \
-p 4000:4000 \
-v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-e AZURE_API_KEY="${AZURE_API_KEY}" \
-e AZURE_API_BASE="${AZURE_API_BASE}" \
"litellm/litellm:${TAG}" \
--config /app/config.yaml --port 4000
- name: Wait for LiteLLM health check
run: |
echo "Waiting for LiteLLM to be ready..."
for i in $(seq 1 30); do
if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
echo "LiteLLM is healthy"
exit 0
fi
echo "Attempt $i/30 - not ready yet, waiting 10s..."
sleep 10
done
echo "LiteLLM failed to start within 5 minutes"
docker logs litellm-rc
exit 1
- name: Start cloudflared tunnel
run: |
# Install cloudflared
curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
chmod +x /usr/local/bin/cloudflared
# Start a quick tunnel (no account needed) and capture the URL
cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
CLOUDFLARED_PID=$!
echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
# Wait for tunnel URL to appear in logs
echo "Waiting for tunnel URL..."
for i in $(seq 1 30); do
TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
if [ -n "$TUNNEL_URL" ]; then
echo "Tunnel URL: $TUNNEL_URL"
echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
exit 0
fi
sleep 2
done
echo "Failed to get tunnel URL"
cat /tmp/cloudflared.log
exit 1
- name: Verify tunnel connectivity
run: |
echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
# Quick tunnels need time for DNS propagation; retry to avoid
# transient NXDOMAIN (curl exit code 6) on first attempt.
for i in $(seq 1 10); do
if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
echo "Tunnel is working (attempt $i)"
exit 0
fi
echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
sleep 5
done
echo "Tunnel failed to become reachable after 50s"
cat /tmp/cloudflared.log
exit 1
- name: Trigger observatory test run
id: trigger
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
run: |
PAYLOAD=$(jq -n \
--arg url "${TUNNEL_URL}" \
--arg key "${LITELLM_MASTER_KEY}" \
'{
deployment_url: $url,
api_key: $key,
test_suite: "TestOAIAzureRelease",
models: ["gpt-4o-mini", "gpt-4o"]
}')
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-H "Content-Type: application/json" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-d "$PAYLOAD")
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)
echo "Response ($HTTP_CODE): $BODY"
if [ "$HTTP_CODE" -ge 400 ]; then
echo "Failed to trigger test run"
exit 1
fi
# Extract request_id for polling this specific run
REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
echo "Failed to extract request_id from response"
exit 1
fi
echo "Request ID: $REQUEST_ID"
echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
- name: Poll for test completion
id: poll
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
run: |
TIMEOUT=900 # 15 minutes
INTERVAL=30
ELAPSED=0
while [ $ELAPSED -lt $TIMEOUT ]; do
STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
echo "Test finished with status: $RUN_STATUS"
echo "$STATUS" > /tmp/observatory_result.json
exit 0
fi
sleep $INTERVAL
ELAPSED=$((ELAPSED + INTERVAL))
done
echo "Timed out waiting for test to complete after ${TIMEOUT}s"
exit 1
- name: Verify test results
run: |
RESULT=$(cat /tmp/observatory_result.json)
echo "Full result: $RESULT"
STATUS=$(echo "$RESULT" | jq -r '.status')
TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
ERROR=$(echo "$RESULT" | jq -r '.error // empty')
echo "Status: $STATUS"
echo "Test passed: $TEST_PASSED"
echo "Failure rate: $FAILURE_RATE"
if [ -n "$ERROR" ]; then
echo "Error: $ERROR"
fi
if [ "$STATUS" = "failed" ]; then
echo "Test run failed"
exit 1
fi
if [ "$TEST_PASSED" != "true" ]; then
echo "Tests did not pass (failure rate: $FAILURE_RATE)"
exit 1
fi
echo "All tests passed!"
- name: Print LiteLLM logs on failure
if: failure()
run: |
docker logs litellm-rc 2>/dev/null || true
cat /tmp/cloudflared.log 2>/dev/null || true
- name: Cleanup
if: always()
run: |
kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
docker rm -f litellm-rc 2>/dev/null || true
.github/workflows/scan_duplicate_issues.yml
+47
View File
@@ -0,0 +1,47 @@
name: Scan Duplicate Issues (One-Time)
on:
workflow_dispatch:
inputs:
threshold:
description: "Similarity threshold (0-1)"
required: false
default: "0.85"
close:
description: "Actually close duplicates (false = dry run)"
required: false
type: boolean
default: false
jobs:
scan:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout scripts
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Scan for duplicate issues
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_THRESHOLD: ${{ inputs.threshold }}
INPUT_CLOSE: ${{ inputs.close }}
run: |
CLOSE_FLAG=""
if [ "$INPUT_CLOSE" = "true" ]; then
CLOSE_FLAG="--close"
fi
python3 .github/scripts/close_duplicate_issues.py \
--scan \
--repo ${{ github.repository }} \
--threshold "$INPUT_THRESHOLD" \
$CLOSE_FLAG
.github/workflows/test-linting.yml
+32
View File
@@ -74,3 +74,35 @@ jobs:
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Run secret scan test
run: |
pip install pytest
pytest tests/litellm/test_no_hardcoded_secrets.py -v
- name: Run ggshield secret scan
env:
GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
run: |
if [ -n "$GITGUARDIAN_API_KEY" ]; then
pip install ggshield
ggshield secret scan repo .
else
echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
fi
.gitignore
+1
View File
@@ -89,6 +89,7 @@ tests/test_custom_dir/*
test.py
litellm_config.yaml
!.github/observatory/litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
AGENTS.md
+41 -1
View File
@@ -177,6 +177,38 @@ When opening issues or pull requests, follow these templates:
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
**Example of BAD** (hardcoded model checks):
```python
@staticmethod
def _is_effort_supported_model(model: str) -> bool:
"""Check if the model supports the output_config.effort parameter..."""
model_lower = model.lower()
if AnthropicConfig._is_claude_4_6_model(model):
return True
return any(
v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
)
```
**Example of GOOD** (config-driven or helper that reads from config):
```python
if (
"claude-3-7-sonnet" in model
or AnthropicConfig._is_claude_4_6_model(model)
or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
):
...
```
Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
## HELPFUL RESOURCES
- Main documentation: https://docs.litellm.ai/
@@ -224,4 +256,12 @@ See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
cd litellm && poetry run ruff check .
```
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
### UI Dashboard development
- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`
Dockerfile
+1 -1
View File
@@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
docs/my-website/docs/observability/datadog.md
+60
View File
@@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics)
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
@@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a
<Image img={require('../../img/dd_llm_obs.png')} />
## Datadog Custom Metrics
| Feature | Details |
|---------|---------|
| **What is logged** | Latency metrics, request counts by status code |
| **Events** | Success + Failure |
| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) |
Publishes the following metrics to Datadog via the `/api/v2/series` endpoint:
| Metric | Type | Description |
|--------|------|-------------|
| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) |
| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) |
| `litellm.llm_api.request_count` | Count | Request count, tagged with status code |
Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`.
All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`.
**Step 1**: Create a `config.yaml` file
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
success_callback: ["datadog_metrics"]
failure_callback: ["datadog_metrics"]
```
**Step 2**: Set required env variables
```shell
DD_API_KEY="your-api-key"
DD_SITE="us5.datadoghq.com" # your datadog site
```
**Step 3**: Start the proxy and make a test request
```shell
litellm --config config.yaml
```
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}]
}'
```
**Step 4**: View metrics in Datadog Metrics Explorer
Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`.
## Datadog Cloud Cost Management
| Feature | Details |
docs/my-website/docs/ocr.md
+91 -7
View File
@@ -61,6 +61,52 @@ async def test_async_ocr():
asyncio.run(test_async_ocr())
```
### Using Local Files
LiteLLM can read local files directly — no manual base64 encoding needed:
```python
from litellm import ocr
# OCR with a local PDF file path
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": "/path/to/document.pdf"
}
)
# OCR with a file object
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": open("document.pdf", "rb")
}
)
# OCR with raw bytes
with open("document.pdf", "rb") as f:
pdf_bytes = f.read()
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": pdf_bytes,
"mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths)
}
)
```
The `file` field accepts:
- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
### Using Base64 Encoded Documents
```python
@@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test request
**Test request — JSON body**
```bash
curl http://0.0.0.0:4000/v1/ocr \
@@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
}'
```
**Test request — multipart file upload**
Upload a file directly using multipart form data. No need to base64-encode the file yourself.
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@/path/to/document.pdf"
```
You can also pass optional parameters as additional form fields:
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@screenshot.png" \
-F 'pages=[0,1,2]' \
-F "include_image_base64=true"
```
## **Request/Response Format**
@@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
| `document` | object | Yes | Document to process. Must contain `type` and URL field |
| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
| `pages` | array | No | List of specific page indices to process (0-indexed) |
| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
| `image_limit` | integer | No | Maximum number of images to return |
@@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
#### Document Format Examples
**For PDFs and documents:**
**For PDFs and documents (URL):**
```json
{
"type": "document_url",
@@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For images:**
**For images (URL):**
```json
{
"type": "image_url",
@@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For local files (SDK):**
```python
{"type": "file", "file": "/path/to/document.pdf"}
{"type": "file", "file": open("image.png", "rb")}
{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
```
**For file uploads (Proxy — multipart form):**
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@document.pdf"
```
### Response Format
The response follows Mistral's OCR format with the following structure:
docs/my-website/docs/pass_through/assembly_ai.md
+144 -35
View File
@@ -1,31 +1,36 @@
# Assembly AI
# AssemblyAI
Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation).
Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation).
| Feature | Supported | Notes |
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ✅ | works across all integrations |
| Logging | ✅ | works across all integrations |
Supports **ALL** Assembly AI Endpoints
Supports **ALL** AssemblyAI Endpoints
[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference)
[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference)
<iframe width="840" height="500" src="https://www.loom.com/embed/aac3f4d74592448992254bfa79b9f62d?sid=267cd0ab-d92b-42fa-b97a-9f385ef8930c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Supported Routes
| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL |
|-------------------|---------------|---------------------|
| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` |
| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` |
## Quick Start
Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
1. Add Assembly AI API Key to your environment
1. Add AssemblyAI API Key to your environment
```bash
export ASSEMBLYAI_API_KEY=""
```
2. Start LiteLLM Proxy
2. Start LiteLLM Proxy
```bash
litellm
@@ -33,53 +38,157 @@ litellm
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
3. Test it!
Let's call the Assembly AI `/v2/transcripts` endpoint
Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on.
```python
import assemblyai as aai
LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
aai.settings.base_url = LITELLM_PROXY_BASE_URL
# Use a publicly-accessible URL
audio_file = "https://assembly.ai/wildfires.mp3"
# URL of the file to transcribe
FILE_URL = "https://assembly.ai/wildfires.mp3"
# Or use a local file:
# audio_file = "./example.mp3"
# You can also transcribe a local file by passing in a file path
# FILE_URL = './path/to/file.mp3'
config = aai.TranscriptionConfig(
speech_models=["universal-3-pro", "universal-2"],
language_detection=True,
speaker_labels=True,
# Speech understanding features
# sentiment_analysis=True,
# entity_detection=True,
# auto_chapters=True,
# summarization=True,
# summary_type=aai.SummarizationType.bullets,
# redact_pii=True,
# content_safety=True,
)
transcriber = aai.Transcriber()
transcript = transcriber.transcribe(FILE_URL)
print(transcript)
print(transcript.id)
transcript = aai.Transcriber().transcribe(audio_file, config=config)
if transcript.status == aai.TranscriptStatus.error:
raise RuntimeError(f"Transcription failed: {transcript.error}")
print(f"\nFull Transcript:\n\n{transcript.text}")
# Optionally print speaker diarization results
# for utterance in transcript.utterances:
# print(f"Speaker {utterance.speaker}: {utterance.text}")
```
## Calling Assembly AI EU endpoints
4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional)
If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
```python
import assemblyai as aai
aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3"
config = aai.TranscriptionConfig(
speech_models=["universal-3-pro", "universal-2"],
language_detection=True,
prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)",
)
transcript = aai.Transcriber().transcribe(audio_file, config)
print(transcript.text)
```
## Calling AssemblyAI EU endpoints
If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
```python
import assemblyai as aai
LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
aai.settings.base_url = LITELLM_PROXY_BASE_URL
# Use a publicly-accessible URL
audio_file = "https://assembly.ai/wildfires.mp3"
# URL of the file to transcribe
FILE_URL = "https://assembly.ai/wildfires.mp3"
# You can also transcribe a local file by passing in a file path
# FILE_URL = './path/to/file.mp3'
# Or use a local file:
# audio_file = "./path/to/file.mp3"
transcriber = aai.Transcriber()
transcript = transcriber.transcribe(FILE_URL)
transcript = transcriber.transcribe(audio_file)
print(transcript)
print(transcript.id)
```
## LLM Gateway
Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support.
[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models)
### Usage
#### LiteLLM Python SDK
```python
import litellm
import os
os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key"
response = litellm.completion(
model="assemblyai/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "What is the capital of France?"}]
)
print(response.choices[0].message.content)
```
#### LiteLLM Proxy
1. Config
```yaml
model_list:
- model_name: assemblyai/*
litellm_params:
model: assemblyai/*
api_key: os.environ/ASSEMBLYAI_API_KEY
```
2. Start proxy
```bash
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```python
import requests
headers = {
"authorization": "Bearer sk-1234" # Bearer <your-virtual-key>
}
response = requests.post(
"http://0.0.0.0:4000/v1/chat/completions",
headers=headers,
json={
"model": "assemblyai/claude-sonnet-4-5-20250929",
"messages": [
{"role": "user", "content": "What is the capital of France?"}
],
"max_tokens": 1000
}
)
result = response.json()
print(result["choices"][0]["message"]["content"])
```
docs/my-website/docs/pass_through/cursor.md
+157
View File
@@ -0,0 +1,157 @@
import Image from '@theme/IdealImage';
# Cursor Cloud Agents
Pass-through endpoints for the [Cursor Cloud Agents API](https://docs.cursor.com/account/api) — launch and manage cloud agents that work on your repositories, in native format (no translation).
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Logged as $0.00 (subscription-based, no per-request pricing) |
| Logging | ✅ | All requests logged with operation classification |
| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
| Streaming | ❌ | Cursor API does not use streaming |
Just replace `https://api.cursor.com` with `LITELLM_PROXY_BASE_URL/cursor` 🚀
**Supported endpoints:**
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/v0/agents` | GET | List agents |
| `/v0/agents` | POST | Launch an agent |
| `/v0/agents/{id}` | GET | Agent status |
| `/v0/agents/{id}` | DELETE | Delete an agent |
| `/v0/agents/{id}/conversation` | GET | Agent conversation |
| `/v0/agents/{id}/followup` | POST | Add follow-up |
| `/v0/agents/{id}/stop` | POST | Stop an agent |
| `/v0/me` | GET | API key info |
| `/v0/models` | GET | List models |
| `/v0/repositories` | GET | List GitHub repositories |
## Quick Start
### 1. Add Cursor API Key on the UI
Navigate to **Models + Endpoints → LLM Credentials** and click **Add Credential**. Select **Cursor** from the provider dropdown — you'll see the Cursor logo. Enter your API key from [cursor.com/settings](https://cursor.com/settings).
<Image img={require('../../img/cursor_add_credential.png')} alt="Add Cursor credential with logo" style={{maxWidth: '800px'}} />
### 2. Launch a Cursor Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents \
-H "Authorization: Bearer <your-litellm-key>" \
-H "Content-Type: application/json" \
-d '{
"prompt": {
"text": "Add a README.md with installation instructions"
},
"source": {
"repository": "https://github.com/your-org/your-repo",
"ref": "main"
},
"target": {
"autoCreatePr": true
}
}'
```
**Expected Response:**
```json
{
"id": "bc_abc123",
"name": "Add README Documentation",
"status": "CREATING",
"source": {
"repository": "https://github.com/your-org/your-repo",
"ref": "main"
},
"target": {
"branchName": "cursor/add-readme-1234",
"url": "https://cursor.com/agents?id=bc_abc123",
"autoCreatePr": true
},
"createdAt": "2024-01-15T10:30:00Z"
}
```
### 3. View Logs
Navigate to **Logs** in the sidebar. Filter by "cursor" to see your agent requests. Each request shows the operation type (e.g., `cursor/cursor:agent:create`), status, duration, and cost.
<Image img={require('../../img/cursor_logs.png')} alt="Cursor requests in Logs page" style={{maxWidth: '800px'}} />
Click on any log entry to see full request details including provider, API base, and metadata.
<Image img={require('../../img/cursor_log_detail.png')} alt="Cursor log entry detail" style={{maxWidth: '800px'}} />
## Examples
Anything after `http://0.0.0.0:4000/cursor` is treated as a provider-specific route, and handled accordingly.
| **Original Endpoint** | **Replace With** |
|---|---|
| `https://api.cursor.com` | `http://0.0.0.0:4000/cursor` (LITELLM_PROXY_BASE_URL) |
| `-u YOUR_API_KEY:` (Basic Auth) | `-H "Authorization: Bearer <your-litellm-key>"` (LiteLLM Virtual Key) |
### List Available Models
```bash
curl http://0.0.0.0:4000/cursor/v0/models \
-H "Authorization: Bearer <your-litellm-key>"
```
### Check Agent Status
```bash
curl http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
-H "Authorization: Bearer <your-litellm-key>"
```
### List All Agents
```bash
curl http://0.0.0.0:4000/cursor/v0/agents \
-H "Authorization: Bearer <your-litellm-key>"
```
### Add Follow-up to Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/followup \
-H "Authorization: Bearer <your-litellm-key>" \
-H "Content-Type: application/json" \
-d '{
"prompt": {
"text": "Also add a section about troubleshooting"
}
}'
```
### Stop an Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/stop \
-H "Authorization: Bearer <your-litellm-key>"
```
### Delete an Agent
```bash
curl -X DELETE http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
-H "Authorization: Bearer <your-litellm-key>"
```
### Get API Key Info
```bash
curl http://0.0.0.0:4000/cursor/v0/me \
-H "Authorization: Bearer <your-litellm-key>"
```
## Related
- [Cursor Cloud Agents API Docs](https://docs.cursor.com/account/api)
- [Pass-through Endpoints Overview](./intro.md)
- [Virtual Keys](../proxy/virtual_keys.md)
docs/my-website/docs/providers/anthropic.md
+3
View File
@@ -417,7 +417,10 @@ print(response)
| Model Name | Function Call |
|------------------|--------------------------------------------|
| claude-opus-4-6 | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4-5 | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4-5 | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4-1 | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
docs/my-website/docs/providers/bedrock.md
+2 -2
View File
@@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4)
- **Computer Use Tools** - AI that can interact with computer interfaces
- **Token-Efficient Tools** - More efficient tool usage patterns
- **Extended Output** - Up to 128K output tokens
@@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic
| Beta Feature | Header Value | Compatible Models | Description |
|--------------|-------------|------------------|-------------|
| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window |
| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |
docs/my-website/docs/proxy/budget_reset_and_tz.md
+27 -20
View File
@@ -1,16 +1,20 @@
## Budget Reset Times and Timezones
# Budget Reset Times and Timezones
LiteLLM now supports predictable budget reset times that align with natural calendar boundaries:
LiteLLM supports predictable budget reset times that align with natural calendar boundaries.
- All budgets reset at midnight (00:00:00) in the configured timezone
- Special handling for common durations:
- Daily (24h/1d): Reset at midnight every day
- Weekly (7d): Reset on Monday at midnight
- Monthly (30d): Reset on the 1st of each month at midnight
## How Budget Resets Work
### Configuring the Timezone
All budgets reset at midnight (00:00:00) in the configured timezone with special handling for common durations:
You can specify the timezone for all budget resets in your configuration file:
| Duration | Reset Behavior |
| --- | --- |
| Daily (24h/1d) | Resets at midnight every day |
| Weekly (7d) | Resets on Monday at midnight |
| Monthly (30d) | Resets on the 1st of each month at midnight |
## Configuring the Timezone
Specify the timezone for all budget resets in your configuration file:
```yaml
litellm_settings:
@@ -19,18 +23,21 @@ litellm_settings:
timezone: "US/Eastern" # Any valid timezone string
```
This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC.
If no timezone is specified, UTC will be used by default.
This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC. If no timezone is specified, UTC will be used by default.
## Supported Timezones
Any valid [IANA timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) is supported (powered by Python's `zoneinfo` module). DST transitions are handled automatically.
Common timezone values:
**Common timezone values:**
- `UTC` - Coordinated Universal Time
- `US/Eastern` - Eastern Time
- `US/Pacific` - Pacific Time
- `Europe/London` - UK Time
- `Asia/Kolkata` - Indian Standard Time (IST)
- `Asia/Bangkok` - Indochina Time (ICT)
- `Asia/Tokyo` - Japan Standard Time
- `Australia/Sydney` - Australian Eastern Time
| Timezone | Description |
| --- | --- |
| `UTC` | Coordinated Universal Time |
| `US/Eastern` | Eastern Time |
| `US/Pacific` | Pacific Time |
| `Europe/London` | UK Time |
| `Asia/Kolkata` | Indian Standard Time (IST) |
| `Asia/Bangkok` | Indochina Time (ICT) |
| `Asia/Tokyo` | Japan Standard Time |
| `Australia/Sydney` | Australian Eastern Time |
docs/my-website/docs/proxy/cli_sso.md
+4
View File
@@ -52,6 +52,10 @@ LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --confi
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
:::note[Experimental UI Session]
When `EXPERIMENTAL_UI_LOGIN` is enabled, the **browser UI login** session uses a fixed 10-minute expiry (not configurable). `LITELLM_UI_SESSION_DURATION` applies only to non-experimental flows.
:::
:::tip
You can check your current token's age and expiration status using:
```bash
docs/my-website/docs/proxy/config_settings.md
+9
View File
@@ -196,6 +196,7 @@ router_settings:
| disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. |
| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
@@ -487,6 +488,7 @@ router_settings:
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
| COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3
| CURSOR_API_BASE | API base URL for Cursor AI provider integration. Default is https://api.cursor.com
| DATABASE_HOST | Hostname for the database server
| DATABASE_NAME | Name of the database
| DATABASE_PASSWORD | Password for the database user
@@ -555,6 +557,10 @@ router_settings:
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
| LITELLM_MCP_CLIENT_TIMEOUT | MCP client connection timeout in seconds (stdio and HTTP/SSE transports). Default is 60
| LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30
| LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10
| LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
@@ -775,6 +781,7 @@ router_settings:
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
| LITELLM_UI_SESSION_DURATION | Duration for UI login session (username/password, SSO, invitation links). Format: "30s", "30m", "24h", "7d". Does not apply to EXPERIMENTAL_UI_LOGIN flow, which uses a fixed 10-minute expiry for security. Default is "24h"
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
@@ -797,6 +804,7 @@ router_settings:
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
| LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@@ -805,6 +813,7 @@ router_settings:
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages`
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
docs/my-website/docs/proxy/health.md
+16
View File
@@ -330,6 +330,22 @@ model_list:
health_check_timeout: 10 # 👈 OVERRIDE HEALTH CHECK TIMEOUT
```
## Health Check Max Tokens
By default, health checks use `max_tokens=1` to minimize cost and latency. For wildcard models, the default is `max_tokens=10`.
You can override this per-model by setting `health_check_max_tokens` in the `model_info` section of your config.yaml.
```yaml
model_list:
- model_name: openai/gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
model_info:
health_check_max_tokens: 5 # 👈 OVERRIDE HEALTH CHECK MAX TOKENS
```
## `/health/readiness`
Unprotected endpoint for checking if proxy is ready to accept requests
docs/my-website/docs/proxy/prometheus.md
+25
View File
@@ -113,6 +113,31 @@ litellm_settings:
```
## Pod Health Metrics
Use these to measure per-pod queue depth and diagnose latency that occurs **before** LiteLLM starts processing a request.
| Metric Name | Type | Description |
|---|---|---|
| `litellm_in_flight_requests` | Gauge | Number of HTTP requests currently in-flight on this uvicorn worker. Tracks the pod's queue depth in real time. With multiple workers, values are summed across all live workers (`livesum`). |
### When to use this
LiteLLM measures latency from when its handler starts. If a request waits in uvicorn's event loop before the handler runs, that wait is invisible to LiteLLM's own logs. `litellm_in_flight_requests` shows how loaded the pod was at any point in time.
```
high in_flight_requests + high ALB TargetResponseTime → pod overloaded, scale out
low in_flight_requests + high ALB TargetResponseTime → delay is pre-ASGI (event loop blocking)
```
You can also check the current value directly without Prometheus:
```bash
curl http://localhost:4000/health/backlog \
-H "Authorization: Bearer sk-..."
# {"in_flight_requests": 47}
```
## Proxy Level Tracking Metrics
Use this to track overall LiteLLM Proxy usage.
docs/my-website/docs/proxy/ui_project_management.md
+142
View File
@@ -0,0 +1,142 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# [Beta] Project Management UI
Manage projects directly from the LiteLLM Admin UI. Projects sit between teams and keys in your organizational hierarchy, enabling fine-grained access control and budget management for specific use cases or applications.
:::info
Project Management is a beta feature. The API and UI are subject to change. For the full API documentation, see [Project Management](./project_management.md).
:::
## Overview
Projects enable you to:
- Organize API keys by use case or application
- Set project-level budgets and rate limits
- Track spend and usage at the project level
- Control which models each project can access
- Maintain clear separation between different applications or teams
**Hierarchy**: `Organizations > Teams > Projects > Keys`
For detailed information about the project API and configuration, see [Project Management](./project_management.md).
## Prerequisites
- Admin or Team Admin access
- At least one team created (projects belong to teams)
- The LiteLLM Admin UI running locally or remote
## Enable Projects in UI Settings
Before you can create projects, you need to enable the Projects feature in the Admin UI settings.
### Step 1: Access Admin Settings
Navigate to the Admin UI (e.g., `http://localhost:4000/ui/?login=success`).
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/b8de4dbf-a23b-4979-84a3-95fe17427b5a/ascreenshot_84dcb13b57a84fd589dff2d5af58adde_text_export.jpeg)
### Step 2: Open Settings Menu
Click the **"New"** button in the top navigation.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/b8de4dbf-a23b-4979-84a3-95fe17427b5a/ascreenshot_447c8ea124f64d0eb18d3c9621f7cbbc_text_export.jpeg)
### Step 3: Navigate to Admin Settings
Click **"Admin Settings"**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/cc2ce9d9-d2d2-49f3-9fb8-c546fb8dfdcf/ascreenshot_fd792e9dbda24e7eb5cdb508c4f181f8_text_export.jpeg)
### Step 4: Open UI Settings
Click **"UI Settings New"**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/d667f4b4-300b-47c6-9d76-12e439519da6/ascreenshot_3f3db4df432843a48b53ae16b311e7df_text_export.jpeg)
### Step 5: Enable Projects Feature
Click the toggle to enable the Projects feature.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/4819f76b-4855-4f5c-8c4b-b4c272399724/ascreenshot_9df0555ae6db425ab839d73485ee9b99_text_export.jpeg)
Once enabled, the Projects section will appear in your Admin UI navigation, and you'll be able to create and manage projects.
## Create and Manage Projects
After enabling the Projects feature, you can create projects from the Projects page.
### Step 1: Navigate to Projects
Click **"Projects New"** in the sidebar.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/889e2e55-af7a-42f1-90d5-8bba8efaa986/ascreenshot_c42e33e2226c4e8b8e8ea83a7c8955e4_text_export.jpeg)
### Step 2: Create a New Project
Click **"Create Project"**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/8ecb531c-8e96-443d-ba1d-1a9e04ba2da3/ascreenshot_74f1b3c1c1b84517ae51881a050df73a_text_export.jpeg)
### Step 3: Enter Project Name
Click the **"Project Name"** field and enter a name for your project.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/83bf0612-2b19-4b28-ae02-bdb122dca4fa/ascreenshot_16ca328a71f04a79bb9641ab9c1ed6fe_text_export.jpeg)
### Step 4: Select a Team
Choose which team this project belongs to. Projects are scoped to teams, so you can only access models and features available to that team.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/653c2f1e-5140-49b8-962f-a2b112f4834c/ascreenshot_7861310ad77d4859adcae789a9d51bd0_text_export.jpeg)
### Step 5: Configure Model Access
Select which models this project has access to. Available models are scoped to the team's allowed models.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/401a5716-ea16-4744-866a-d0ed6007065d/ascreenshot_a936c3ca417a49b2b603c890dee9d0ea_text_export.jpeg)
### Step 6: Create Project
Click **"Create Project"** to save your project.
![](https://colony-recorder.s3.amazonaws.com/files/2026-03-01/2f9f9ba1-df0b-4bef-b17c-77dfc38372f7/ascreenshot_933e4c1b119d43beb84161b94b17b764_text_export.jpeg)
## Use Cases
### Key Organization Within Teams
Organize API keys within a team by use case or application. Group related keys together in projects so you can manage budgets, model access, and permissions as a unit instead of individually.
### Cost Allocation
Assign projects to different cost centers or teams. Track spend per project and allocate costs back to the responsible team or business unit.
### Feature Rollout
Create a dedicated project for new features or experimental use cases. Control which models are available and set conservative rate limits during testing.
### Customer Segmentation
If you're a platform, create projects for different customer segments or use cases. Control resource allocation independently for each segment.
## Next Steps
After creating a project:
1. **Generate API Keys** – Create API keys scoped to your project for application use
2. **Set Budgets** – Configure project-level budget limits via the [Project Management API](./project_management.md)
3. **Track Spend** – View project-level spend in the Usage dashboard
4. **Manage Access** – Use [Access Groups](./access_groups.md) to control model and MCP server access
## Related Documentation
- [Project Management API](./project_management.md) – Full API reference for projects
- [Access Groups](./access_groups.md) – Define reusable access controls for models, MCP servers, and agents
- [Virtual Keys](./virtual_keys.md) – Create and manage API keys scoped to projects
- [Role-based Access Control](./access_control.md) – Organizations, teams, and user roles
- [Spend Logs](./spend_logs_deletion.md) – Track detailed request-level costs and usage
docs/my-website/docs/troubleshoot/latency_overhead.md
+33 -1
View File
@@ -2,9 +2,41 @@
Use this guide when you see unexpected latency overhead between LiteLLM proxy and the LLM provider.
## The Invisible Latency Gap
LiteLLM measures latency from when its handler starts. If a request waits in uvicorn's event loop **before** the handler runs, that wait is invisible to LiteLLM's own logs.
```
T=0 Request arrives at load balancer
[queue wait — LiteLLM never logs this]
T=10 LiteLLM handler starts → timer begins
T=20 Response sent
LiteLLM logs: 10s User experiences: 20s
```
To measure the pre-handler wait, poll `/health/backlog` on each pod:
```bash
curl http://localhost:4000/health/backlog \
-H "Authorization: Bearer sk-..."
# {"in_flight_requests": 47}
```
Or scrape the `litellm_in_flight_requests` Prometheus gauge at `/metrics`.
| `in_flight_requests` | ALB `TargetResponseTime` | Diagnosis |
|---|---|---|
| High | High | Pod overloaded → scale out |
| Low | High | Delay is pre-ASGI — check for sync blocking code or event loop saturation |
| High | Normal | Pod is busy but healthy, no queue buildup |
If you're on **AWS ALB**, correlate `litellm_in_flight_requests` spikes with ALB's `TargetResponseTime` CloudWatch metric. The gap between what ALB reports and what LiteLLM logs is the invisible wait.
## Quick Checklist
1. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. Start here.
1. **Check `in_flight_requests` on each pod** via `/health/backlog` or the `litellm_in_flight_requests` Prometheus gauge — this tells you if requests are queuing before LiteLLM starts processing. Start here for unexplained latency.
2. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request.
2. **Is DEBUG logging enabled?** This is the #1 cause of latency with large payloads.
3. **Are you sending large base64 payloads?** (images, PDFs) — see [Large Payload Overhead](#large-payload-overhead).
4. **Enable detailed timing headers** to pinpoint where time is spent.
docs/my-website/docs/tutorials/fallbacks.md
+4
View File
@@ -2,6 +2,10 @@
This tutorial demonstrates how to employ the `completion()` function with model fallbacks to ensure reliability. LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls
## Set Up Fallbacks for a Virtual Key
<iframe width="840" height="500" src="https://www.loom.com/embed/35539129dd104313aff40eb1cd255778" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Usage
To use fallback models with `completion()`, specify a list of models in the `fallbacks` parameter.
docs/my-website/img/cursor_add_credential.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 28 KiB

docs/my-website/img/cursor_log_detail.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

docs/my-website/img/cursor_logs.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 45 KiB

docs/my-website/package-lock.json
Generated
+5905 -1163
View File
File diff suppressed because it is too large Load Diff
docs/my-website/package.json
+6 -5
View File
@@ -15,10 +15,10 @@
},
"dependencies": {
"@docusaurus/core": "3.8.1",
"@docusaurus/plugin-google-gtag": "3.8.1",
"@docusaurus/plugin-google-gtag": "^3.5.2",
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@docusaurus/preset-classic": "^3.5.2",
"@docusaurus/theme-mermaid": "^3.5.2",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
@@ -62,9 +62,10 @@
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.8",
"minimatch": ">=10.2.1",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
"serialize-javascript": ">=7.0.3",
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1",
"lodash-es": ">=4.17.23",
@@ -94,4 +95,4 @@
"serve-static": ">=1.16.0",
"path-to-regexp": ">=0.1.12"
}
}
}
docs/my-website/release_notes/v1.81.14.md
+65
View File
@@ -489,6 +489,71 @@ graph LR
---
## Security
We run [Grype](https://github.com/anchore/grype) and [Trivy](https://github.com/aquasecurity/trivy) security scans on every LiteLLM Docker image. Here's the vulnerability report for this release across all published images:
### Docker Image Scan Summary
| Image | Critical | High | Medium | Low |
|-------|----------|------|--------|-----|
| `ghcr.io/berriai/litellm:main-latest` | **0** ✅ | 4 unique CVEs | 4 | 1 |
| `ghcr.io/berriai/litellm-ee:main-latest` | **0** ✅ | 4 unique CVEs | 4 | 1 |
| `ghcr.io/berriai/litellm-non_root:main-latest` | **1** | 11 unique CVEs | 6 | 2 |
| `ghcr.io/berriai/litellm-database:main-latest` | **1** | 7 unique CVEs | 5 | 1 |
| `ghcr.io/berriai/litellm-spend_logs:main-latest` | **4** | 35 matches | 40 | 10 |
:::note
Vulnerability counts are based on full image scans including build-time tooling. High match counts are often inflated by packages like `minimatch` appearing at multiple versions; the unique CVE counts above reflect the actual distinct vulnerabilities.
:::
### Critical Severity
**1. Node.js Critical (non-root, database, spend_logs images):**
Node.js 24.12.0 is used **only** for the Admin UI build and Prisma client generation — it is **not** part of the LiteLLM Python application runtime.
| Package | Vulnerability | Description | Fix Version |
|---------|---------------|-------------|-------------|
| `node` | CVE-2025-55130 | Node.js critical vulnerability | 20.20.0 |
**2. OpenSSL & Go Critical (spend_logs image only):**
The `spend_logs` image contains additional vulnerabilities in the underlying Go modules and system libraries.
| Package | Vulnerability | Description | Fix Version |
|---------|---------------|-------------|-------------|
| `libcrypto3`, `libssl3` | CVE-2025-15467 | OpenSSL critical vulnerability | 3.3.6-r0 |
| `stdlib` (Go) | CVE-2025-68121 | Go standard library critical vulnerability | 1.24.13+ |
### High Severity
All high-severity vulnerabilities are in **npm/Node.js build-time dependencies** or system-level libraries — they are **not** in the LiteLLM Python application code.
**Present in all images:**
| Package | Vulnerability | Description | Fix Version |
|---------|---------------|-------------|-------------|
| `minimatch` | CVE-2026-26996 | DoS via specially crafted glob patterns | 10.2.1+ / 9.0.6+ |
| `minimatch` | CVE-2026-27903 | DoS due to unbounded recursive backtracking | 10.2.3+ / 9.0.7+ |
| `minimatch` | CVE-2026-27904 | DoS via catastrophic backtracking in glob expressions | 10.2.3+ / 9.0.7+ |
| `tar` | CVE-2026-26960 / GHSA-83g3-92jg-28cx | Arbitrary file read/write via malicious archive hardlinks | 7.5.8 |
### Medium Severity (all images)
| Package | Vulnerability | Status |
|---------|---------------|--------|
| `pypdf` 6.7.2 | GHSA-x7hp-r3qg-r3cj | Fix available in 6.7.3 |
| Python 3.13 | CVE-2025-15366, CVE-2025-15367, CVE-2025-12781 | No upstream fix available |
### Recommendations
- **LiteLLM Main & EE images** (`litellm:main-latest`, `litellm-ee:main-latest`) have the best security posture with **0 critical vulnerabilities**.
- All HIGH/CRITICAL findings in the main images relate to build-time Node.js/npm tooling, not the Python runtime.
- We are actively monitoring upstream Python and system library fixes for remaining medium-severity vulnerabilities.
To report a security vulnerability, email support@berri.ai with details and steps to reproduce.
---
## Documentation Updates
- Add OpenAI Agents SDK with LiteLLM guide - [PR #21311](https://github.com/BerriAI/litellm/pull/21311)
docs/my-website/release_notes/v1.82.0.md
+467
View File
@@ -0,0 +1,467 @@
---
title: "v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
slug: "v1-82-0"
date: 2026-02-28T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-1.82.0
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.82.0
```
</TabItem>
</Tabs>
## Key Highlights
- **Realtime API guardrails** — [Full guardrails support for `/v1/realtime` WebSocket sessions with pre/post-call enforcement, voice transcription hooks, session termination policies, and Vertex AI Gemini Live support](../../docs/proxy/guardrails) - [PR #22152](https://github.com/BerriAI/litellm/pull/22152), [PR #22153](https://github.com/BerriAI/litellm/pull/22153), [PR #22161](https://github.com/BerriAI/litellm/pull/22161), [PR #22165](https://github.com/BerriAI/litellm/pull/22165)
- **Projects Management** — [New Projects UI with full CRUD, project-scoped virtual keys, and admin opt-in toggle — organize teams and keys by project](../../docs/proxy/ui_store_model_db_setting) - [PR #22315](https://github.com/BerriAI/litellm/pull/22315), [PR #22360](https://github.com/BerriAI/litellm/pull/22360), [PR #22373](https://github.com/BerriAI/litellm/pull/22373), [PR #22412](https://github.com/BerriAI/litellm/pull/22412)
- **Guardrail ecosystem expansion** — [Noma v2, Lakera v2 post-call, Singapore regulatory policies (PDPA + MAS), employment discrimination blockers, code execution blocker, guardrail policy versioning, and production monitoring](../../docs/proxy/guardrails) - [PR #21400](https://github.com/BerriAI/litellm/pull/21400), [PR #21783](https://github.com/BerriAI/litellm/pull/21783), [PR #21948](https://github.com/BerriAI/litellm/pull/21948)
- **OpenAI Codex 5.3 — day 0** — [Full support for `gpt-5.3-codex` on OpenAI and Azure, plus `gpt-audio-1.5` and `gpt-realtime-1.5` model coverage](../../docs/providers/openai) - [PR #22035](https://github.com/BerriAI/litellm/pull/22035)
- **10+ performance optimizations** — Streaming hot-path fixes, Redis pipeline batching, database task batching, ModelResponse init skip, and router cache improvements — lower latency and CPU on every request
---
## New Models / Updated Models
#### New Model Support (20 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| OpenAI | `gpt-5.3-codex` | 272K | $1.75 | $14.00 | Reasoning, coding |
| Azure OpenAI | `azure/gpt-5.3-codex` | 272K | $1.75 | $14.00 | Azure deployment |
| OpenAI | `gpt-audio-1.5` | 128K | $2.50 | $10.00 | Audio model |
| Azure OpenAI | `azure/gpt-audio-1.5-2026-02-23` | 128K | $2.50 | $10.00 | Audio model |
| OpenAI | `gpt-realtime-1.5` | 32K | $4.00 | $16.00 | Realtime model |
| Azure OpenAI | `azure/gpt-realtime-1.5-2026-02-23` | 32K | $4.00 | $16.00 | Realtime model |
| Groq | `groq/openai/gpt-oss-safeguard-20b` | 131K | $0.075 | $0.30 | Guardrail inference |
| Google Vertex AI | `vertex_ai/gemini-3.1-flash-image-preview` | - | - | - | Image generation |
| Perplexity | `perplexity/perplexity/sonar` | - | - | - | Sonar search |
| Perplexity | `perplexity/openai/gpt-5.1` | - | - | - | Hosted routing |
| Perplexity | `perplexity/openai/gpt-5-mini` | - | - | - | Hosted routing |
| Perplexity | `perplexity/google/gemini-2.5-flash` | - | - | - | Hosted routing |
| Perplexity | `perplexity/google/gemini-2.5-pro` | - | - | - | Hosted routing |
| Perplexity | `perplexity/google/gemini-3-flash-preview` | - | - | - | Hosted routing |
| Perplexity | `perplexity/google/gemini-3-pro-preview` | - | - | - | Hosted routing |
| Perplexity | `perplexity/anthropic/claude-haiku-4-5` | - | - | - | Hosted routing |
| Perplexity | `perplexity/anthropic/claude-sonnet-4-5` | - | - | - | Hosted routing |
| Perplexity | `perplexity/anthropic/claude-opus-4-5` | - | - | - | Hosted routing |
| Perplexity | `perplexity/anthropic/claude-opus-4-6` | - | - | - | Hosted routing |
| Perplexity | `perplexity/xai/grok-4-1-fast-non-reasoning` | - | - | - | Hosted routing |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Day 0 support for `gpt-5.3-codex` on OpenAI and Azure - [PR #22035](https://github.com/BerriAI/litellm/pull/22035)
- Add `gpt-audio-1.5` model cost map - [PR #22303](https://github.com/BerriAI/litellm/pull/22303)
- Add `gpt-realtime-1.5` model cost map - [PR #22304](https://github.com/BerriAI/litellm/pull/22304)
- Add `audio` as supported OpenAI param - [PR #22092](https://github.com/BerriAI/litellm/pull/22092)
- Add `prompt_cache_key` and `prompt_cache_retention` support - [PR #20397](https://github.com/BerriAI/litellm/pull/20397)
- **[Azure OpenAI](../../docs/providers/azure)**
- New Azure OpenAI models 2026-02-25 - [PR #22114](https://github.com/BerriAI/litellm/pull/22114)
- **[Anthropic](../../docs/providers/anthropic)**
- Add v1 Anthropic Responses API transformation - [PR #22087](https://github.com/BerriAI/litellm/pull/22087)
- Sanitize `tool_use` IDs in `convert_to_anthropic_tool_invoke` - [PR #21964](https://github.com/BerriAI/litellm/pull/21964)
- Fix model wildcard access issue - [PR #21917](https://github.com/BerriAI/litellm/pull/21917)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Encode model ARNs for OpenAI-compatible Bedrock imported models - [PR #21701](https://github.com/BerriAI/litellm/pull/21701)
- Support optional regional STS endpoint in role assumption - [PR #21640](https://github.com/BerriAI/litellm/pull/21640)
- Native structured outputs API support - [PR #21222](https://github.com/BerriAI/litellm/pull/21222)
- **[Google Vertex AI](../../docs/providers/vertex)**
- Add `gemini-3.1-flash-image-preview` to model cost map - [PR #22223](https://github.com/BerriAI/litellm/pull/22223)
- Enable `context-1m-2025-08-07` beta header for Vertex AI provider - [PR #21867](https://github.com/BerriAI/litellm/pull/21867)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add OpenRouter native models to model cost map - [PR #20520](https://github.com/BerriAI/litellm/pull/20520)
- Add OpenRouter Opus 4.6 to model map - [PR #20525](https://github.com/BerriAI/litellm/pull/20525)
- **[Mistral](../../docs/providers/mistral)**
- Adjust `mistral-small-2503` input/output cost per token - [PR #22097](https://github.com/BerriAI/litellm/pull/22097)
- **[Groq](../../docs/providers/groq)**
- Add `groq/openai/gpt-oss-safeguard-20b` model pricing - [PR #21951](https://github.com/BerriAI/litellm/pull/21951)
- **[AI/ML](../../docs/providers/aiml)**
- Update AIML model pricing - [PR #22139](https://github.com/BerriAI/litellm/pull/22139)
- **[Ollama](../../docs/providers/ollama)**
- Thread `api_base` to `get_model_info` + graceful fallback - [PR #21970](https://github.com/BerriAI/litellm/pull/21970)
- **[PublicAI](../../docs/providers/openai)**
- Fix function calling for PublicAI Apertus models - [PR #21582](https://github.com/BerriAI/litellm/pull/21582)
- **[xAI](../../docs/providers/xai)**
- Add deprecation dates for `grok-2-vision-1212` and `grok-3-mini` models - [PR #20102](https://github.com/BerriAI/litellm/pull/20102)
- **General**
- Forward auth headers of provider - [PR #22070](https://github.com/BerriAI/litellm/pull/22070)
- Normalize camelCase `thinking` param keys to snake_case - [PR #21762](https://github.com/BerriAI/litellm/pull/21762)
- Allow `dimensions` param passthrough for non-text-embedding-3 OpenAI models - [PR #22144](https://github.com/BerriAI/litellm/pull/22144)
### Bug Fixes
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Fix converse handling for `parallel_tool_calls` - [PR #22267](https://github.com/BerriAI/litellm/pull/22267)
- Restore `parallel_tool_calls` mapping in `map_openai_params` - [PR #22333](https://github.com/BerriAI/litellm/pull/22333)
- Correct `modelInput` format for Converse API batch models - [PR #21656](https://github.com/BerriAI/litellm/pull/21656)
- Prevent double UUID in `create_file` S3 key - [PR #21650](https://github.com/BerriAI/litellm/pull/21650)
- Filter internal `json_tool_call` when mixed with real tools - [PR #21107](https://github.com/BerriAI/litellm/pull/21107)
- Pass timeout param to Bedrock rerank HTTP client - [PR #22021](https://github.com/BerriAI/litellm/pull/22021)
- **[Anthropic](../../docs/providers/anthropic)**
- Fix model cost map for anthropic fast and `inference_geo` - [PR #21904](https://github.com/BerriAI/litellm/pull/21904)
- **[Image Generation](../../docs/image_generation)**
- Propagate `extra_headers` to upstream image generation - [PR #22026](https://github.com/BerriAI/litellm/pull/22026)
- Add `ChatCompletionImageObject` in `OpenAIChatCompletionAssistantMessage` - [PR #22155](https://github.com/BerriAI/litellm/pull/22155)
- **General**
- Preserve forwarding of server-side called tools - [PR #22260](https://github.com/BerriAI/litellm/pull/22260)
- Fix free model handling from UI paths - [PR #22258](https://github.com/BerriAI/litellm/pull/22258)
- Fix `None` TypeError in mapping - [PR #22080](https://github.com/BerriAI/litellm/pull/22080)
---
## LLM API Endpoints
#### Features
- **[Realtime API](../../docs/response_api)**
- Guardrails support for `/v1/realtime` WebSocket endpoint - [PR #22152](https://github.com/BerriAI/litellm/pull/22152)
- Vertex AI Gemini Live via unified `/realtime` endpoint - [PR #22153](https://github.com/BerriAI/litellm/pull/22153)
- Guardrails with `pre_call`/`post_call` mode on realtime WebSocket - [PR #22161](https://github.com/BerriAI/litellm/pull/22161)
- `end_session_after_n_fails` + Endpoint Settings wizard step - [PR #22165](https://github.com/BerriAI/litellm/pull/22165)
- Guardrail hook for voice transcription - [PR #21976](https://github.com/BerriAI/litellm/pull/21976)
- Fix guardrails not firing for Gemini/Vertex AI and `provider_config` realtime sessions - [PR #22168](https://github.com/BerriAI/litellm/pull/22168)
- Add logging, spend tracking support + tool tracing - [PR #22105](https://github.com/BerriAI/litellm/pull/22105)
- **[Video Generation](../../docs/video_generation)**
- Add `variant` parameter to video content download - [PR #21955](https://github.com/BerriAI/litellm/pull/21955)
- Pass `api_key` from `litellm_params` to video remix handlers - [PR #21965](https://github.com/BerriAI/litellm/pull/21965)
- Apply custom video pricing from deployment `model_info` - [PR #21923](https://github.com/BerriAI/litellm/pull/21923)
- Fix passing of image and parameters in videos API - [PR #22170](https://github.com/BerriAI/litellm/pull/22170)
- **[OCR](../../docs/providers/openai#ocr--document-understanding)**
- Enable local file support for OCR - [PR #22133](https://github.com/BerriAI/litellm/pull/22133)
- **[Websearch / Tool Calling](../../docs/completion/input)**
- Preserve thinking blocks in agentic loop follow-up messages - [PR #21604](https://github.com/BerriAI/litellm/pull/21604)
- **General**
- Add configurable upper bound for chunk processing time - [PR #22209](https://github.com/BerriAI/litellm/pull/22209)
- Emit `x-litellm-overhead-duration-ms` header for streaming requests - [PR #22027](https://github.com/BerriAI/litellm/pull/22027)
#### Bugs
- **General**
- Fix mypy attr-defined errors on realtime websocket calls - [PR #22202](https://github.com/BerriAI/litellm/pull/22202)
---
## Management Endpoints / UI
#### Features
- **Projects**
- Add Projects page with list and create flows - [PR #22315](https://github.com/BerriAI/litellm/pull/22315)
- Add Project Details page with edit modal - [PR #22360](https://github.com/BerriAI/litellm/pull/22360)
- Add project keys table and project dropdown on key create/edit - [PR #22373](https://github.com/BerriAI/litellm/pull/22373)
- Add delete project action to Projects table - [PR #22412](https://github.com/BerriAI/litellm/pull/22412)
- Add Projects Opt-In Toggle in Admin Settings - [PR #22416](https://github.com/BerriAI/litellm/pull/22416)
- Include `created_at` and `updated_at` in `/project/list` response - [PR #22323](https://github.com/BerriAI/litellm/pull/22323)
- Add tags in project - [PR #22216](https://github.com/BerriAI/litellm/pull/22216)
- **Virtual Keys + Access Groups**
- Add bidirectional team/key sync for Access Group CRUD flows - [PR #22253](https://github.com/BerriAI/litellm/pull/22253)
- Add pagination and search to `/key/aliases` to prevent OOMs - [PR #22137](https://github.com/BerriAI/litellm/pull/22137)
- Add paginated key alias selector in UI - [PR #22157](https://github.com/BerriAI/litellm/pull/22157)
- Add `project_id` and `access_group_id` filters for key list endpoint - [PR #22356](https://github.com/BerriAI/litellm/pull/22356)
- Add KeyInfoHeader component - [PR #22047](https://github.com/BerriAI/litellm/pull/22047)
- Restrict Edit Settings to key owners - [PR #21985](https://github.com/BerriAI/litellm/pull/21985)
- Fix virtual key grace period from env/UI - [PR #20321](https://github.com/BerriAI/litellm/pull/20321)
- **Agents**
- Assign virtual keys to agents - [PR #22045](https://github.com/BerriAI/litellm/pull/22045)
- Assign tools to agents - [PR #22064](https://github.com/BerriAI/litellm/pull/22064)
- Ensure internal users cannot create agents (RBAC enforcement) - [PR #22329](https://github.com/BerriAI/litellm/pull/22329)
- **Proxy Auth / SSO**
- OIDC discovery URLs, roles array handling, and dot-notation error hints - [PR #22336](https://github.com/BerriAI/litellm/pull/22336)
- Add PROXY_ADMIN role to system user for key rotation - [PR #21896](https://github.com/BerriAI/litellm/pull/21896)
- **Usage / Spend Logs**
- Add user filtering to usage page - [PR #22059](https://github.com/BerriAI/litellm/pull/22059)
- Allow using AI to understand usage patterns - [PR #22042](https://github.com/BerriAI/litellm/pull/22042)
- Use backend `request_duration_ms` and make Duration sortable in Logs - [PR #22122](https://github.com/BerriAI/litellm/pull/22122)
- Add `request_duration_ms` to SpendLogs - [PR #22066](https://github.com/BerriAI/litellm/pull/22066)
- Enrich failure spend logs with key/team metadata - [PR #22049](https://github.com/BerriAI/litellm/pull/22049)
- Show real tool names in logs for Anthropic-format tools - [PR #22048](https://github.com/BerriAI/litellm/pull/22048)
- **Models + Endpoints**
- Show proxy URL in ModelHub - [PR #21660](https://github.com/BerriAI/litellm/pull/21660)
- Add `/public/endpoints` for provider endpoint support - [PR #22248](https://github.com/BerriAI/litellm/pull/22248)
- **UI Improvements**
- Add custom favicon support - [PR #21653](https://github.com/BerriAI/litellm/pull/21653)
- Add Blog Dropdown in Navbar - [PR #21859](https://github.com/BerriAI/litellm/pull/21859)
- Add UI banner warning for detailed debug mode - [PR #21527](https://github.com/BerriAI/litellm/pull/21527)
- Make auth value optional for MCP Server create flow - [PR #22119](https://github.com/BerriAI/litellm/pull/22119)
- Tool policies: auto-discover tools + policy enforcement guardrail - [PR #22041](https://github.com/BerriAI/litellm/pull/22041)
- **Health Checks**
- Add health check max tokens configuration - [PR #22299](https://github.com/BerriAI/litellm/pull/22299)
- Limit concurrent health checks with `health_check_concurrency` - [PR #20584](https://github.com/BerriAI/litellm/pull/20584)
- Fix health check `model_id` filtering - [PR #21071](https://github.com/BerriAI/litellm/pull/21071)
#### Bugs
- Populate `user_id` and `user_info` for admin users in `/user/info` - [PR #22239](https://github.com/BerriAI/litellm/pull/22239)
- Fix virtual keys pagination stale totals when filtering - [PR #22222](https://github.com/BerriAI/litellm/pull/22222)
- Fix Spend Update Queue aggregation never triggers with default presets - [PR #21963](https://github.com/BerriAI/litellm/pull/21963)
- Fix timezone config lookup and replace hardcoded timezone map with `ZoneInfo` - [PR #21754](https://github.com/BerriAI/litellm/pull/21754)
- Fix custom auth budget issue - [PR #22164](https://github.com/BerriAI/litellm/pull/22164)
- Fix missing OAuth session state - [PR #21992](https://github.com/BerriAI/litellm/pull/21992)
- Fix Transport Type for OpenAPI Spec on UI - [PR #22005](https://github.com/BerriAI/litellm/pull/22005)
- Fix Claude Code plugin schema - [PR #22271](https://github.com/BerriAI/litellm/pull/22271)
- Add missing migration for `LiteLLM_ClaudeCodePluginTable` - [PR #22335](https://github.com/BerriAI/litellm/pull/22335)
- Only tag selected deployment in access group creation - [PR #21655](https://github.com/BerriAI/litellm/pull/21655)
- State management fixes for CheckBatchCost - [PR #21921](https://github.com/BerriAI/litellm/pull/21921)
- Remove duplicate antd import in ToolPolicies - [PR #22107](https://github.com/BerriAI/litellm/pull/22107)
---
## AI Integrations
### Logging
- **[DataDog](../../docs/proxy/logging#datadog)**
- Add ability to trace metrics in DataDog - [PR #22103](https://github.com/BerriAI/litellm/pull/22103)
- Correlate LiteLLM call IDs with DataDog APM spans - [PR #22219](https://github.com/BerriAI/litellm/pull/22219)
- Fix TTS metric emission issues - [PR #20632](https://github.com/BerriAI/litellm/pull/20632)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Add opt-in `stream` label on `litellm_proxy_total_requests_metric` - [PR #22023](https://github.com/BerriAI/litellm/pull/22023)
- Fix team `+Inf` budgets in Prometheus metrics - [PR #22243](https://github.com/BerriAI/litellm/pull/22243)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Fix Langfuse OTEL trace issues - [PR #21309](https://github.com/BerriAI/litellm/pull/21309)
- **[Arize Phoenix](../../docs/observability/arize_phoenix)**
- Fix nested traces coexistence with OTEL callback - [PR #22169](https://github.com/BerriAI/litellm/pull/22169)
- **[Slack](../../docs/proxy/alerting)**
- Add optional digest mode for Slack alert types - [PR #21683](https://github.com/BerriAI/litellm/pull/21683)
- **General**
- Fix Gemini trace ID missing in logging - [PR #22077](https://github.com/BerriAI/litellm/pull/22077)
- Populate `cache_read_input_tokens` from `prompt_tokens_details` for OpenAI/Azure - [PR #22090](https://github.com/BerriAI/litellm/pull/22090)
### Guardrails
- **[Noma](../../docs/proxy/guardrails)**
- Noma guardrails v2 based on custom guardrails framework - [PR #21400](https://github.com/BerriAI/litellm/pull/21400)
- **[LakeraAI](../../docs/proxy/guardrails)**
- Add Lakera v2 post-call hook with fixed PII masking - [PR #21783](https://github.com/BerriAI/litellm/pull/21783)
- **[Presidio](../../docs/proxy/guardrails)**
- Fix Presidio streaming and false positives - [PR #21949](https://github.com/BerriAI/litellm/pull/21949)
- Fix Presidio streaming v3 reliability improvements - [PR #22283](https://github.com/BerriAI/litellm/pull/22283)
- Prevent Presidio crash on non-JSON responses - [PR #22084](https://github.com/BerriAI/litellm/pull/22084)
- **Built-in Guardrails**
- Block code execution guardrail to prevent agents from executing code - [PR #22154](https://github.com/BerriAI/litellm/pull/22154)
- Employment discrimination topic blockers for 5 protected classes - [PR #21962](https://github.com/BerriAI/litellm/pull/21962)
- Claims agent guardrails (5 categories + policy template) - [PR #22113](https://github.com/BerriAI/litellm/pull/22113)
- New code execution evaluation dataset - [PR #22065](https://github.com/BerriAI/litellm/pull/22065)
- Tool policies: auto-discover tools + policy enforcement - [PR #22041](https://github.com/BerriAI/litellm/pull/22041)
- **Policy Templates**
- Singapore guardrail policies (PDPA + MAS AI Risk Management) - [PR #21948](https://github.com/BerriAI/litellm/pull/21948)
- Prefix SG guardrail policy IDs with country code - [PR #21974](https://github.com/BerriAI/litellm/pull/21974)
- Guardrail policy versioning - [PR #21862](https://github.com/BerriAI/litellm/pull/21862)
- **Guardrail Monitoring**
- Guardrail Monitor — measure guardrail reliability in production - [PR #21944](https://github.com/BerriAI/litellm/pull/21944)
- **Security**
- Fix unauthenticated RCE and sandbox escape in custom code guardrail - [PR #22095](https://github.com/BerriAI/litellm/pull/22095)
### Prompt Management
No major prompt management changes in this release.
### Secret Managers
No major secret manager changes in this release.
---
## Spend Tracking, Budgets and Rate Limiting
- **Priority PayGo cost tracking** for Gemini/Vertex AI - [PR #21909](https://github.com/BerriAI/litellm/pull/21909)
- **Add `request_duration_ms` to SpendLogs** for latency tracking per request - [PR #22066](https://github.com/BerriAI/litellm/pull/22066)
- **Add `in_flight_requests` metric** to `/health/backlog` + Prometheus - [PR #22319](https://github.com/BerriAI/litellm/pull/22319)
- **Enrich failure spend logs** with key/team metadata - [PR #22049](https://github.com/BerriAI/litellm/pull/22049)
- **Add spend tracking lifecycle logging** for debugging spend flows - [PR #22029](https://github.com/BerriAI/litellm/pull/22029)
- **Fix budget timezone config lookup** and replace hardcoded timezone map with `ZoneInfo` - [PR #21754](https://github.com/BerriAI/litellm/pull/21754)
- **Fix Spend Update Queue aggregation** never triggering with default presets - [PR #21963](https://github.com/BerriAI/litellm/pull/21963)
- **Avoid mutating caller-owned dicts** in `SpendUpdateQueue` aggregation - [PR #21742](https://github.com/BerriAI/litellm/pull/21742)
- **Optimize old spendlog deletion** cron job - [PR #21930](https://github.com/BerriAI/litellm/pull/21930)
- **Health check max tokens** configuration - [PR #22299](https://github.com/BerriAI/litellm/pull/22299)
---
## MCP Gateway
- **Pass MCP auth headers** from request context to tool fetch for `/v1/responses` and `/chat/completions` - [PR #22291](https://github.com/BerriAI/litellm/pull/22291)
- **Default `available_on_public_internet` to true** for MCP server behavior consistency - [PR #22331](https://github.com/BerriAI/litellm/pull/22331)
- **Clear error messages** for IP filtering / no available tools - [PR #22142](https://github.com/BerriAI/litellm/pull/22142)
- **Strip stale `mcp-session-id` header** to prevent 400 errors across proxy workers - [PR #21417](https://github.com/BerriAI/litellm/pull/21417)
- **Skip health check for MCP** with passthrough token auth - [PR #21982](https://github.com/BerriAI/litellm/pull/21982)
- **Fix missing OAuth session state** - [PR #21992](https://github.com/BerriAI/litellm/pull/21992)
- **Fix Transport Type** for OpenAPI Spec on UI - [PR #22005](https://github.com/BerriAI/litellm/pull/22005)
- **Add e2e test** for stateless StreamableHTTP behavior - [PR #22033](https://github.com/BerriAI/litellm/pull/22033)
---
## Performance / Loadbalancing / Reliability improvements
**Streaming & hot-path**
- Streaming latency improvements — 4 targeted hot-path fixes - [PR #22346](https://github.com/BerriAI/litellm/pull/22346)
- Skip throwaway `Usage()` construction in `ModelResponse.__init__` - [PR #21611](https://github.com/BerriAI/litellm/pull/21611)
- Optimize `is_model_o_series_model` with `startswith` - [PR #21690](https://github.com/BerriAI/litellm/pull/21690)
- Use cached `_safe_get_request_headers` instead of per-request construction - [PR #21430](https://github.com/BerriAI/litellm/pull/21430)
- Emit `x-litellm-overhead-duration-ms` header for streaming requests - [PR #22027](https://github.com/BerriAI/litellm/pull/22027)
**Database & Redis**
- Batch 11 `create_task()` calls into 1 in `update_database()` - [PR #22028](https://github.com/BerriAI/litellm/pull/22028)
- Redis pipeline spend updates for batched writes - [PR #22044](https://github.com/BerriAI/litellm/pull/22044)
- Recover from prisma-query-engine zombie process - [PR #21899](https://github.com/BerriAI/litellm/pull/21899)
- Optimize old spendlog deletion cron job - [PR #21930](https://github.com/BerriAI/litellm/pull/21930)
**Router & caching**
- Add cache invalidation for `_cached_get_model_group_info` - [PR #20376](https://github.com/BerriAI/litellm/pull/20376)
- Remove cache eviction close that kills in-use httpx clients - [PR #22247](https://github.com/BerriAI/litellm/pull/22247)
- Store background task references in `LLMClientCache._remove_key` to prevent unawaited coroutine warnings - [PR #22143](https://github.com/BerriAI/litellm/pull/22143)
- Fix `ensure_arrival_time` set before calculating queue time - [PR #21918](https://github.com/BerriAI/litellm/pull/21918)
**Connection management**
- Only set `enable_cleanup_closed` on aiohttp when required - [PR #21897](https://github.com/BerriAI/litellm/pull/21897)
- Prometheus child_exit cleanup for gunicorn workers - [PR #22324](https://github.com/BerriAI/litellm/pull/22324)
- Prometheus multiprocess cleanup - [PR #22221](https://github.com/BerriAI/litellm/pull/22221)
- Limit concurrent health checks with `health_check_concurrency` - [PR #20584](https://github.com/BerriAI/litellm/pull/20584)
- Isolate `get_config` failures from model sync loop - [PR #22224](https://github.com/BerriAI/litellm/pull/22224)
**Other**
- Semantic cache: support configurable vector dimensions - [PR #21649](https://github.com/BerriAI/litellm/pull/21649)
- Honor `MAX_STRING_LENGTH_PROMPT_IN_DB` from config env vars - [PR #22106](https://github.com/BerriAI/litellm/pull/22106)
- Enhance `MidStreamFallbackError` to preserve original status code and attributes - [PR #22225](https://github.com/BerriAI/litellm/pull/22225)
- Network mock utility for testing - [PR #21942](https://github.com/BerriAI/litellm/pull/21942)
- Add missing return type annotations to iterator protocol methods in streaming_handler - [PR #21750](https://github.com/BerriAI/litellm/pull/21750)
---
## Security
- Fix critical/high CVEs in OS-level libs and NPM transitive dependencies - [PR #22008](https://github.com/BerriAI/litellm/pull/22008)
- Fix unauthenticated RCE and sandbox escape in custom code guardrail - [PR #22095](https://github.com/BerriAI/litellm/pull/22095)
- Remove hardcoded base64 string flagged by secret scanner - [PR #22125](https://github.com/BerriAI/litellm/pull/22125)
---
## Documentation Updates
- Add OpenAI Agents SDK tutorial with LiteLLM Proxy - [PR #21221](https://github.com/BerriAI/litellm/pull/21221)
- Add OpenClaw integration tutorial - [PR #21605](https://github.com/BerriAI/litellm/pull/21605)
- Add Google GenAI SDK tutorial (JS & Python) - [PR #21885](https://github.com/BerriAI/litellm/pull/21885)
- Add Gollem Go agent framework cookbook example - [PR #21747](https://github.com/BerriAI/litellm/pull/21747)
- Update AssemblyAI docs with Universal-3 Pro, Speech Understanding, and LLM Gateway - [PR #21130](https://github.com/BerriAI/litellm/pull/21130)
- Add `store_model_in_db` release docs - [PR #21863](https://github.com/BerriAI/litellm/pull/21863)
- Add Credential Usage Tracking docs - [PR #22112](https://github.com/BerriAI/litellm/pull/22112)
- Add proxy request tags docs - [PR #22129](https://github.com/BerriAI/litellm/pull/22129)
- Add trailing slash to `/mcp` endpoint URLs - [PR #20509](https://github.com/BerriAI/litellm/pull/20509)
- Add pre-PR checklist to UI contributing guide - [PR #21886](https://github.com/BerriAI/litellm/pull/21886)
- Replace Azure OpenAI key with mock key in docs - [PR #21997](https://github.com/BerriAI/litellm/pull/21997)
- Add performance & reliability section to v1.81.14 release notes - [PR #21950](https://github.com/BerriAI/litellm/pull/21950)
- Update v1.81.12-stable release notes to point to stable.1 - [PR #22036](https://github.com/BerriAI/litellm/pull/22036)
- Add security vulnerability scan report to v1.81.14 release notes - [PR #22385](https://github.com/BerriAI/litellm/pull/22385)
---
## New Contributors
* @janfrederickk made their first contribution in [PR #21660](https://github.com/BerriAI/litellm/pull/21660)
* @hztBUAA made their first contribution in [PR #21656](https://github.com/BerriAI/litellm/pull/21656)
* @LeeJuOh made their first contribution in [PR #21754](https://github.com/BerriAI/litellm/pull/21754)
* @WhoisMonesh made their first contribution in [PR #21750](https://github.com/BerriAI/litellm/pull/21750)
* @trevorprater made their first contribution in [PR #21747](https://github.com/BerriAI/litellm/pull/21747)
* @edwiniac made their first contribution in [PR #21870](https://github.com/BerriAI/litellm/pull/21870)
* @stakeswky made their first contribution in [PR #21867](https://github.com/BerriAI/litellm/pull/21867)
* @ta-stripe made their first contribution in [PR #21701](https://github.com/BerriAI/litellm/pull/21701)
* @ron-zhong made their first contribution in [PR #21948](https://github.com/BerriAI/litellm/pull/21948)
* @Arindam200 made their first contribution in [PR #21221](https://github.com/BerriAI/litellm/pull/21221)
* @Canvinus made their first contribution in [PR #21964](https://github.com/BerriAI/litellm/pull/21964)
* @nicolopignatelli made their first contribution in [PR #21951](https://github.com/BerriAI/litellm/pull/21951)
* @MarshHawk made their first contribution in [PR #20584](https://github.com/BerriAI/litellm/pull/20584)
* @gavksingh made their first contribution in [PR #22106](https://github.com/BerriAI/litellm/pull/22106)
* @roni-frantchi made their first contribution in [PR #22090](https://github.com/BerriAI/litellm/pull/22090)
* @noahnistler made their first contribution in [PR #22133](https://github.com/BerriAI/litellm/pull/22133)
* @dylan-duan-aai made their first contribution in [PR #21130](https://github.com/BerriAI/litellm/pull/21130)
* @rasmi made their first contribution in [PR #22322](https://github.com/BerriAI/litellm/pull/22322)
---
## Diff Summary
## 02/28/2026
* New Models / Updated Models: 26
* LLM API Endpoints: 14
* Management Endpoints / UI: 38
* AI Integrations: 25
* Spend Tracking, Budgets and Rate Limiting: 10
* MCP Gateway: 8
* Performance / Loadbalancing / Reliability improvements: 22
* Security: 3
* Documentation Updates: 14
---
## Full Changelog
[v1.81.14.rc.1...v1.82.0](https://github.com/BerriAI/litellm/compare/v1.81.14.rc.1...v1.82.0)
docs/my-website/sidebars.js
+2
View File
@@ -348,6 +348,7 @@ const sidebars = {
"proxy/access_control",
"proxy/self_serve",
"proxy/public_teams",
"proxy/ui_project_management",
"proxy/ui/bulk_edit_users",
"proxy/ui/page_visibility",
]
@@ -635,6 +636,7 @@ const sidebars = {
"pass_through/bedrock",
"pass_through/azure_passthrough",
"pass_through/cohere",
"pass_through/cursor",
"pass_through/google_ai_studio",
"pass_through/langfuse",
"pass_through/mistral",
enterprise/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.32"
version = "0.1.33"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.32"
version = "0.1.33"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql
+3
View File
@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "agent_id" TEXT;
litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql
+18
View File
@@ -0,0 +1,18 @@
-- CreateTable
CREATE TABLE "LiteLLM_ClaudeCodePluginTable" (
"id" TEXT NOT NULL,
"name" TEXT NOT NULL,
"version" TEXT,
"description" TEXT,
"manifest_json" TEXT,
"files_json" TEXT DEFAULT '{}',
"enabled" BOOLEAN NOT NULL DEFAULT true,
"created_at" TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
CONSTRAINT "LiteLLM_ClaudeCodePluginTable_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ClaudeCodePluginTable_name_key" ON "LiteLLM_ClaudeCodePluginTable"("name");
litellm-proxy-extras/litellm_proxy_extras/migrations/20260228100000_add_spend_logs_composite_index/migration.sql
+2
View File
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX "LiteLLM_SpendLogs_startTime_request_id_idx" ON "LiteLLM_SpendLogs"("startTime", "request_id");
litellm-proxy-extras/litellm_proxy_extras/migrations/20260228110000_mcp_default_public_internet_true/migration.sql
+2
View File
@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ALTER COLUMN "available_on_public_internet" SET DEFAULT true;
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+19 -2
View File
@@ -300,7 +300,7 @@ model LiteLLM_MCPServerTable {
token_url String?
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(false)
available_on_public_internet Boolean @default(true)
}
// Generate Tokens for Proxy
@@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken {
config Json @default("{}")
user_id String?
team_id String?
agent_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
@@ -504,6 +505,7 @@ model LiteLLM_SpendLogs {
agent_id String?
proxy_server_request Json? @default("{}")
@@index([startTime])
@@index([startTime, request_id])
@@index([end_user])
@@index([session_id])
}
@@ -1094,4 +1096,19 @@ model LiteLLM_AccessGroupTable {
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
}
// Claude Code Plugin Marketplace table
model LiteLLM_ClaudeCodePluginTable {
id String @id @default(uuid())
name String @unique
version String?
description String?
manifest_json String?
files_json String? @default("{}")
enabled Boolean @default(true)
created_at DateTime? @default(now())
updated_at DateTime? @default(now()) @updatedAt
created_by String?
@@map("LiteLLM_ClaudeCodePluginTable")
}
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.48"
version = "0.4.50"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.48"
version = "0.4.50"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+12 -3
View File
@@ -12,6 +12,13 @@ warnings.filterwarnings(
### INIT VARIABLES #########################
import threading
import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv()
from typing import (
Callable,
List,
@@ -74,12 +81,9 @@ from litellm.constants import (
DEFAULT_ALLOWED_FAILS,
)
import httpx
import dotenv
# register_async_client_cleanup is lazy-loaded and called on first access
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
if litellm_mode == "DEV":
dotenv.load_dotenv()
####################################################
@@ -105,6 +109,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"prometheus",
"otel",
"datadog",
"datadog_metrics",
"datadog_llm_observability",
"galileo",
"braintrust",
@@ -197,6 +202,9 @@ telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
use_chat_completions_url_for_anthropic_messages: bool = bool(
os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API
retry = True
### AUTH ###
api_key: Optional[str] = None
@@ -1513,6 +1521,7 @@ if TYPE_CHECKING:
from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig
from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig
from .llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig as HostedVLLMResponsesAPIConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
litellm/_lazy_imports_registry.py
+5
View File
@@ -226,6 +226,7 @@ LLM_CONFIG_NAMES = (
"AzureOpenAIOSeriesResponsesAPIConfig",
"XAIResponsesAPIConfig",
"LiteLLMProxyResponsesAPIConfig",
"HostedVLLMResponsesAPIConfig",
"VolcEngineResponsesAPIConfig",
"PerplexityResponsesConfig",
"DatabricksResponsesAPIConfig",
@@ -897,6 +898,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.litellm_proxy.responses.transformation",
"LiteLLMProxyResponsesAPIConfig",
),
"HostedVLLMResponsesAPIConfig": (
".llms.hosted_vllm.responses.transformation",
"HostedVLLMResponsesAPIConfig",
),
"VolcEngineResponsesAPIConfig": (
".llms.volcengine.responses.transformation",
"VolcEngineResponsesAPIConfig",
litellm/caching/llm_caching_handler.py
+11 -4
View File
@@ -3,22 +3,29 @@ Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Set
from .in_memory_cache import InMemoryCache
class LLMClientCache(InMemoryCache):
# Background tasks must be stored to prevent garbage collection, which would
# trigger "coroutine was never awaited" warnings. See:
# https://docs.python.org/3/library/asyncio-task.html#creating-tasks
# Intentionally shared across all instances as a global task registry.
_background_tasks: Set[asyncio.Task] = set()
def _remove_key(self, key: str) -> None:
"""Close async clients before evicting them to prevent connection pool leaks."""
value = self.cache_dict.get(key)
super()._remove_key(key)
if value is not None:
close_fn = getattr(value, "aclose", None) or getattr(
value, "close", None
)
close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
if close_fn and asyncio.iscoroutinefunction(close_fn):
try:
asyncio.get_running_loop().create_task(close_fn())
task = asyncio.get_running_loop().create_task(close_fn())
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
except RuntimeError:
pass
elif close_fn and callable(close_fn):
litellm/completion_extras/litellm_responses_transformation/transformation.py
+5 -3
View File
@@ -49,6 +49,7 @@ if TYPE_CHECKING:
ALL_RESPONSES_API_TOOL_PARAMS,
AllMessageValues,
ChatCompletionImageObject,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
OpenAIMessageContentListBlock,
)
@@ -161,7 +162,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content,
content, # type: ignore[arg-type]
role, # type: ignore
),
}
@@ -213,7 +214,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
{
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)),
"content": self._convert_content_to_responses_format(content, cast(str, role)), # type: ignore[arg-type]
}
)
@@ -579,7 +580,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content: Optional[
Union[
str,
Iterable[Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock"]],
List[Any],
Iterable[Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]],
]
],
role: str,
litellm/constants.py
+21 -2
View File
@@ -49,6 +49,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
)
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
# Maximum wall-clock seconds a streaming response is allowed to run.
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
_max_stream_duration_env = os.getenv("LITELLM_MAX_STREAMING_DURATION_SECONDS", None)
LITELLM_MAX_STREAMING_DURATION_SECONDS = (
float(_max_stream_duration_env) if _max_stream_duration_env is not None else None
)
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
# Set to 0 to disable truncation.
@@ -129,6 +137,12 @@ MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache")
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10"))
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
MCP_CLIENT_TIMEOUT = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0"))
MCP_TOOL_LISTING_TIMEOUT = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0"))
MCP_METADATA_TIMEOUT = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0"))
MCP_HEALTH_CHECK_TIMEOUT = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0"))
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@@ -185,9 +199,9 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo
# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300))
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000))
AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50)
os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500)
)
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
@@ -1314,6 +1328,11 @@ CLI_JWT_EXPIRATION_HOURS = int(
or 24
)
########################### UI SESSION DURATION ###########################
# Duration for UI login session (username/password, SSO, invitation links). Format: "30s", "30m", "24h", "7d"
# Does NOT apply to EXPERIMENTAL_UI_LOGIN flow, which intentionally uses a fixed 10-minute expiry for security.
LITELLM_UI_SESSION_DURATION = os.getenv("LITELLM_UI_SESSION_DURATION", "24h")
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
litellm/exceptions.py
+17 -2
View File
@@ -955,7 +955,8 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore
generated_content: str = "",
is_pre_first_chunk: bool = False,
):
self.status_code = 503 # Service Unavailable
original_status = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
self.message = f"litellm.MidStreamFallbackError: {message}"
self.model = model
self.llm_provider = llm_provider
@@ -978,7 +979,14 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore
else:
self.response = response
# Call the parent constructor
# Save the original attributes before they are overridden by ServiceUnavailableError
_saved_response = self.response
_saved_request = getattr(self.response, "request", None) or httpx.Request(
method="POST", url=f"https://{llm_provider}.com/v1/"
)
_saved_message = self.message
# Call the parent constructor (which hardcodes status_code=503 and modifies the response object)
super().__init__(
message=self.message,
llm_provider=llm_provider,
@@ -988,6 +996,13 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore
max_retries=self.max_retries,
num_retries=self.num_retries,
)
# Restore the propagated status and original response/request objects
self.status_code = int(original_status) if original_status is not None else 503
self.response = _saved_response
self.request = _saved_request
self.message = _saved_message
self.args = (_saved_message,)
def __str__(self):
_message = self.message
litellm/experimental_mcp_client/client.py
+3 -2
View File
@@ -30,6 +30,7 @@ from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from litellm._logging import verbose_logger
from litellm.constants import MCP_CLIENT_TIMEOUT
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.types.llms.custom_http import VerifyTypes
from litellm.types.mcp import (
@@ -63,7 +64,7 @@ class MCPClient:
transport_type: MCPTransportType = MCPTransport.http,
auth_type: MCPAuthType = None,
auth_value: Optional[Union[str, Dict[str, str]]] = None,
timeout: float = 60.0,
timeout: Optional[float] = None,
stdio_config: Optional[MCPStdioConfig] = None,
extra_headers: Optional[Dict[str, str]] = None,
ssl_verify: Optional[VerifyTypes] = None,
@@ -71,7 +72,7 @@ class MCPClient:
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
self.auth_type: MCPAuthType = auth_type
self.timeout: float = timeout
self.timeout: float = timeout if timeout is not None else MCP_CLIENT_TIMEOUT
self._mcp_auth_value: Optional[Union[str, Dict[str, str]]] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.extra_headers: Optional[Dict[str, str]] = extra_headers
litellm/images/main.py
+1
View File
@@ -483,6 +483,7 @@ def image_generation( # noqa: PLR0915
organization=organization,
aimg_generation=aimg_generation,
client=client,
headers=headers,
)
elif custom_llm_provider == "bedrock":
if model is None:
litellm/integrations/callback_configs.json
+22 -1
View File
@@ -83,6 +83,27 @@
},
"description": "Datadog Logging Integration"
},
{
"id": "datadog_metrics",
"displayName": "Datadog Metrics",
"logo": "datadog.png",
"supports_key_team_logging": false,
"dynamic_params": {
"dd_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Datadog API key for authentication",
"required": true
},
"dd_site": {
"type": "text",
"ui_name": "Site",
"description": "Datadog site URL (e.g., us5.datadoghq.com)",
"required": true
}
},
"description": "Datadog Custom Metrics Integration"
},
{
"id": "datadog_cost_management",
"displayName": "Datadog Cost Management",
@@ -434,4 +455,4 @@
},
"description": "SQS Queue (AWS) Logging Integration"
}
]
]
litellm/integrations/datadog/datadog_metrics.py
+286
View File
@@ -0,0 +1,286 @@
import asyncio
import gzip
import os
import time
from datetime import datetime
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.datadog.datadog_handler import (
get_datadog_env,
get_datadog_hostname,
get_datadog_pod_name,
get_datadog_service,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.datadog_metrics import (
DatadogMetricPoint,
DatadogMetricSeries,
DatadogMetricsPayload,
)
from litellm.types.utils import StandardLoggingPayload
class DatadogMetricsLogger(CustomBatchLogger):
def __init__(self, start_periodic_flush: bool = True, **kwargs):
self.dd_api_key = os.getenv("DD_API_KEY")
self.dd_app_key = os.getenv("DD_APP_KEY")
self.dd_site = os.getenv("DD_SITE", "datadoghq.com")
if not self.dd_api_key:
verbose_logger.warning(
"Datadog Metrics: DD_API_KEY is required. Integration will not work."
)
self.upload_url = f"https://api.{self.dd_site}/api/v2/series"
self.async_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
# Initialize lock
self.flush_lock = asyncio.Lock()
# Only set flush_lock if not already provided by caller
if "flush_lock" not in kwargs:
kwargs["flush_lock"] = self.flush_lock
# Send metrics more quickly to datadog (every 5 seconds)
if "flush_interval" not in kwargs:
kwargs["flush_interval"] = 5
super().__init__(**kwargs)
# Start periodic flush task only if instructed
if start_periodic_flush:
asyncio.create_task(self.periodic_flush())
def _extract_tags(
self,
log: StandardLoggingPayload,
status_code: Optional[Union[str, int]] = None,
) -> List[str]:
"""
Builds the list of tags for a Datadog metric point
"""
# Base tags
tags = [
f"env:{get_datadog_env()}",
f"service:{get_datadog_service()}",
f"version:{os.getenv('DD_VERSION', 'unknown')}",
f"HOSTNAME:{get_datadog_hostname()}",
f"POD_NAME:{get_datadog_pod_name()}",
]
# Add metric-specific tags
if provider := log.get("custom_llm_provider"):
tags.append(f"provider:{provider}")
if model := log.get("model"):
tags.append(f"model_name:{model}")
if model_group := log.get("model_group"):
tags.append(f"model_group:{model_group}")
if status_code is not None:
tags.append(f"status_code:{status_code}")
# Extract team tag
metadata = log.get("metadata", {}) or {}
team_tag = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias") # type: ignore
or metadata.get("user_api_key_team_id")
or metadata.get("team_id") # type: ignore
)
if team_tag:
tags.append(f"team:{team_tag}")
return tags
def _add_metrics_from_log(
self,
log: StandardLoggingPayload,
kwargs: dict,
status_code: Union[str, int] = "200",
):
"""
Extracts latencies and appends Datadog metric series to the queue
"""
tags = self._extract_tags(log, status_code=status_code)
# We record metrics with the end_time as the timestamp for the point
end_time_dt = kwargs.get("end_time") or datetime.now()
timestamp = int(end_time_dt.timestamp())
# 1. Total Request Latency Metric (End to End)
start_time_dt = kwargs.get("start_time")
if start_time_dt and end_time_dt:
total_duration = (end_time_dt - start_time_dt).total_seconds()
series_total_latency: DatadogMetricSeries = {
"metric": "litellm.request.total_latency",
"type": 3, # gauge
"points": [{"timestamp": timestamp, "value": total_duration}],
"tags": tags,
}
self.log_queue.append(series_total_latency)
# 2. LLM API Latency Metric (Provider alone)
api_call_start_time = kwargs.get("api_call_start_time")
if api_call_start_time and end_time_dt:
llm_api_duration = (end_time_dt - api_call_start_time).total_seconds()
series_llm_latency: DatadogMetricSeries = {
"metric": "litellm.llm_api.latency",
"type": 3, # gauge
"points": [{"timestamp": timestamp, "value": llm_api_duration}],
"tags": tags,
}
self.log_queue.append(series_llm_latency)
# 3. Request Count / Status Code
series_count: DatadogMetricSeries = {
"metric": "litellm.llm_api.request_count",
"type": 1, # count
"points": [{"timestamp": timestamp, "value": 1.0}],
"tags": tags,
"interval": self.flush_interval,
}
self.log_queue.append(series_count)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object", None
)
if standard_logging_object is None:
return
self._add_metrics_from_log(
log=standard_logging_object, kwargs=kwargs, status_code="200"
)
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
except Exception as e:
verbose_logger.exception(
f"Datadog Metrics: Error in async_log_success_event: {str(e)}"
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object", None
)
if standard_logging_object is None:
return
# Extract status code from error information
status_code = "500" # default
error_information = (
standard_logging_object.get("error_information", {}) or {}
)
error_code = error_information.get("error_code") # type: ignore
if error_code is not None:
status_code = str(error_code)
self._add_metrics_from_log(
log=standard_logging_object, kwargs=kwargs, status_code=status_code
)
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
except Exception as e:
verbose_logger.exception(
f"Datadog Metrics: Error in async_log_failure_event: {str(e)}"
)
async def async_send_batch(self):
if not self.log_queue:
return
batch = self.log_queue.copy()
payload_data: DatadogMetricsPayload = {"series": batch}
try:
await self._upload_to_datadog(payload_data)
except Exception as e:
verbose_logger.exception(
f"Datadog Metrics: Error in async_send_batch: {str(e)}"
)
raise
async def _upload_to_datadog(self, payload: DatadogMetricsPayload):
if not self.dd_api_key:
return
headers = {
"Content-Type": "application/json",
"DD-API-KEY": self.dd_api_key,
}
if self.dd_app_key:
headers["DD-APPLICATION-KEY"] = self.dd_app_key
json_data = safe_dumps(payload)
compressed_data = gzip.compress(json_data.encode("utf-8"))
headers["Content-Encoding"] = "gzip"
response = await self.async_client.post(
self.upload_url, content=compressed_data, headers=headers # type: ignore
)
response.raise_for_status()
verbose_logger.debug(
f"Datadog Metrics: Uploaded {len(payload['series'])} metric points. Status: {response.status_code}"
)
async def async_health_check(self) -> IntegrationHealthCheckStatus:
"""
Check if the service is healthy
"""
try:
# Send a test metric point to Datadog
test_metric_point: DatadogMetricPoint = {
"timestamp": int(time.time()),
"value": 1.0,
}
test_metric_series: DatadogMetricSeries = {
"metric": "litellm.health_check",
"type": 3, # Gauge
"points": [test_metric_point],
"tags": ["env:health_check"],
}
payload_data: DatadogMetricsPayload = {"series": [test_metric_series]}
await self._upload_to_datadog(payload_data)
return IntegrationHealthCheckStatus(
status="healthy",
error_message=None,
)
except Exception as e:
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message=str(e),
)
async def get_request_response_payload(
self,
request_id: str,
start_time_utc: Optional[datetime],
end_time_utc: Optional[datetime],
) -> Optional[dict]:
pass
litellm/integrations/prometheus.py
+4
View File
@@ -2686,6 +2686,8 @@ class PrometheusLogger(CustomLogger):
if team_info:
team_object.budget_reset_at = team_info.budget_reset_at
if team_object.max_budget is None and team_info.max_budget is not None:
team_object.max_budget = team_info.max_budget
return team_object
@@ -2903,6 +2905,8 @@ class PrometheusLogger(CustomLogger):
if user_info:
user_object.budget_reset_at = user_info.budget_reset_at
if user_object.max_budget is None and user_info.max_budget is not None:
user_object.max_budget = user_info.max_budget
return user_object
litellm/integrations/websearch_interception/handler.py
+47 -1
View File
@@ -299,12 +299,54 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
)
# Return tools dict with tool calls
# Extract thinking blocks from response content.
# When extended thinking is enabled, the model response includes
# thinking/redacted_thinking blocks that must be preserved and
# prepended to the follow-up assistant message.
thinking_blocks: List[Dict] = []
if isinstance(response, dict):
content = response.get("content", [])
else:
content = getattr(response, "content", []) or []
for block in content:
if isinstance(block, dict):
block_type = block.get("type")
else:
block_type = getattr(block, "type", None)
if block_type in ("thinking", "redacted_thinking"):
if isinstance(block, dict):
thinking_blocks.append(block)
else:
# Convert object to dict using getattr, matching the
# pattern in _detect_from_non_streaming_response
thinking_block_dict: Dict = {"type": block_type}
if block_type == "thinking":
thinking_block_dict["thinking"] = getattr(
block, "thinking", ""
)
thinking_block_dict["signature"] = getattr(
block, "signature", ""
)
else: # redacted_thinking
thinking_block_dict["data"] = getattr(
block, "data", ""
)
thinking_blocks.append(thinking_block_dict)
if thinking_blocks:
verbose_logger.debug(
f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response"
)
# Return tools dict with tool calls and thinking blocks
tools_dict = {
"tool_calls": tool_calls,
"tool_type": "websearch",
"provider": custom_llm_provider,
"response_format": "anthropic",
"thinking_blocks": thinking_blocks,
}
return True, tools_dict
@@ -387,6 +429,7 @@ class WebSearchInterceptionLogger(CustomLogger):
"""
tool_calls = tools["tool_calls"]
thinking_blocks = tools.get("thinking_blocks", [])
verbose_logger.debug(
f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)"
@@ -396,6 +439,7 @@ class WebSearchInterceptionLogger(CustomLogger):
model=model,
messages=messages,
tool_calls=tool_calls,
thinking_blocks=thinking_blocks,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
stream=stream,
@@ -442,6 +486,7 @@ class WebSearchInterceptionLogger(CustomLogger):
model: str,
messages: List[Dict],
tool_calls: List[Dict],
thinking_blocks: List[Dict],
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
stream: bool,
@@ -495,6 +540,7 @@ class WebSearchInterceptionLogger(CustomLogger):
assistant_message, user_message = WebSearchTransformation.transform_response(
tool_calls=tool_calls,
search_results=final_search_results,
thinking_blocks=thinking_blocks,
)
# Make follow-up request with search results
litellm/integrations/websearch_interception/transformation.py
+27 -7
View File
@@ -4,7 +4,7 @@ WebSearch Tool Transformation
Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
"""
import json
from typing import Any, Dict, List, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, Union
from litellm._logging import verbose_logger
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@@ -224,6 +224,7 @@ class WebSearchTransformation:
tool_calls: List[Dict],
search_results: List[str],
response_format: str = "anthropic",
thinking_blocks: Optional[List[Dict]] = None,
) -> Tuple[Dict, Union[Dict, List[Dict]]]:
"""
Transform LiteLLM search results to Anthropic/OpenAI tool_result format.
@@ -235,6 +236,10 @@ class WebSearchTransformation:
tool_calls: List of tool_use/tool_calls dicts from transform_request
search_results: List of search result strings (one per tool_call)
response_format: Response format - "anthropic" or "openai" (default: "anthropic")
thinking_blocks: Optional list of thinking/redacted_thinking blocks
from the model's response. When present, prepended to the
assistant message content (required by Anthropic API when
thinking is enabled).
Returns:
(assistant_message, user_or_tool_messages):
@@ -247,19 +252,29 @@ class WebSearchTransformation:
)
else:
return WebSearchTransformation._transform_response_anthropic(
tool_calls, search_results
tool_calls, search_results, thinking_blocks=thinking_blocks
)
@staticmethod
def _transform_response_anthropic(
tool_calls: List[Dict],
search_results: List[str],
thinking_blocks: Optional[List[Dict]] = None,
) -> Tuple[Dict, Dict]:
"""Transform to Anthropic format (single user message with tool_result blocks)"""
# Build assistant message with tool_use blocks
assistant_message = {
"role": "assistant",
"content": [
# Build assistant message content
assistant_content: List[Dict] = []
# Prepend thinking blocks if present.
# When extended thinking is enabled, Anthropic requires the assistant
# message to start with thinking/redacted_thinking blocks before any
# tool_use blocks. Same pattern as anthropic_messages_pt in factory.py.
if thinking_blocks:
assistant_content.extend(thinking_blocks)
# Add tool_use blocks
assistant_content.extend(
[
{
"type": "tool_use",
"id": tc["id"],
@@ -267,7 +282,12 @@ class WebSearchTransformation:
"input": tc["input"],
}
for tc in tool_calls
],
]
)
assistant_message = {
"role": "assistant",
"content": assistant_content,
}
# Build user message with tool_result blocks
litellm/litellm_core_utils/custom_logger_registry.py
+2
View File
@@ -20,6 +20,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.datadog.datadog import DataDogLogger
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger
from litellm.integrations.deepeval import DeepEvalLogger
from litellm.integrations.dotprompt import DotpromptManager
from litellm.integrations.focus.focus_logger import FocusLogger
@@ -66,6 +67,7 @@ class CustomLoggerRegistry:
"prometheus": PrometheusLogger,
"datadog": DataDogLogger,
"datadog_llm_observability": DataDogLLMObsLogger,
"datadog_metrics": DatadogMetricsLogger,
"gcs_bucket": GCSBucketLogger,
"opik": OpikLogger,
"argilla": ArgillaLogger,
litellm/litellm_core_utils/dd_tracing.py
+46 -1
View File
@@ -5,7 +5,7 @@ If the ddtrace package is not installed, the tracer will be a no-op.
"""
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from litellm.secret_managers.main import get_secret_bool
@@ -76,3 +76,48 @@ if should_use_dd_tracer:
tracer = NullTracer()
else:
tracer = NullTracer()
def get_active_span() -> Optional[Any]:
"""
Return the active Datadog span, checking current span first and then root span.
"""
try:
current_span_fn = getattr(tracer, "current_span", None)
if callable(current_span_fn):
current_span = current_span_fn()
if current_span is not None:
return current_span
current_root_span_fn = getattr(tracer, "current_root_span", None)
if callable(current_root_span_fn):
return current_root_span_fn()
except Exception:
return None
return None
def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
"""
Best-effort helper to set a tag on the active Datadog span.
Returns:
bool: True if a span tag was set, False otherwise.
"""
if not tag_key or tag_value is None:
return False
span = get_active_span()
if span is None:
return False
try:
if hasattr(span, "set_tag_str"):
span.set_tag_str(tag_key, str(tag_value))
return True
if hasattr(span, "set_tag"):
span.set_tag(tag_key, str(tag_value))
return True
except Exception:
return False
return False
litellm/litellm_core_utils/health_check_helpers.py
+5 -4
View File
@@ -14,7 +14,6 @@ TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9U
class HealthCheckHelpers:
@staticmethod
async def ahealth_check_wildcard_models(
model: str,
@@ -44,7 +43,9 @@ class HealthCheckHelpers:
model_params["model"] = cheapest_models[0]
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["max_tokens"] = 10 # gpt-5-nano throws errors for max_tokens=1
model_params["max_tokens"] = model_params.get(
"max_tokens", 10
) # gpt-5-nano throws errors for max_tokens=1
await acompletion(**model_params)
return {}
@@ -130,7 +131,7 @@ class HealthCheckHelpers:
Callable,
]:
"""
Returns a dictionary of mode handlers for health check calls.
Returns a dictionary of mode handlers for health check calls.
Mode Handlers are Callables that need to be run for execution of the health check call.
@@ -215,4 +216,4 @@ class HealthCheckHelpers:
"document_url": TEST_PDF_URL,
},
),
}
}
litellm/litellm_core_utils/litellm_logging.py
+13
View File
@@ -133,6 +133,7 @@ from ..integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from ..integrations.custom_prompt_management import CustomPromptManagement
from ..integrations.datadog.datadog import DataDogLogger
from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger
from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from ..integrations.dotprompt import DotpromptManager
from ..integrations.dynamodb import DyanmoDBLogger
@@ -3661,6 +3662,14 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_datadog_logger = DataDogLogger()
_in_memory_loggers.append(_datadog_logger)
return _datadog_logger # type: ignore
elif logging_integration == "datadog_metrics":
for callback in _in_memory_loggers:
if isinstance(callback, DatadogMetricsLogger):
return callback # type: ignore
_datadog_metrics_logger = DatadogMetricsLogger()
_in_memory_loggers.append(_datadog_metrics_logger)
return _datadog_metrics_logger # type: ignore
elif logging_integration == "datadog_llm_observability":
_datadog_llm_obs_logger = DataDogLLMObsLogger()
_in_memory_loggers.append(_datadog_llm_obs_logger)
@@ -4268,6 +4277,10 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, DataDogLogger):
return callback
elif logging_integration == "datadog_metrics":
for callback in _in_memory_loggers:
if isinstance(callback, DatadogMetricsLogger):
return callback
elif logging_integration == "datadog_llm_observability":
for callback in _in_memory_loggers:
if isinstance(callback, DataDogLLMObsLogger):
litellm/litellm_core_utils/llm_cost_calc/utils.py
+60
View File
@@ -8,9 +8,11 @@ from litellm._logging import verbose_logger
from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
CompletionTokensDetailsWrapper,
ImageResponse,
ModelInfo,
PassthroughCallTypes,
PromptTokensDetailsWrapper,
ServiceTier,
Usage,
)
@@ -767,6 +769,64 @@ def generic_cost_per_token( # noqa: PLR0915
return prompt_cost, completion_cost
def calculate_image_response_cost_from_usage(
model: str,
image_response: ImageResponse,
custom_llm_provider: str,
) -> Optional[float]:
"""
Calculate image generation cost from usage metadata when available.
Returns:
Optional[float]: total cost from token usage, or None when usage metadata
is missing/incomplete and caller should fall back to flat per-image pricing.
"""
usage = image_response.usage
if usage is None:
return None
prompt_tokens = usage.input_tokens
completion_tokens = usage.output_tokens
total_tokens = usage.total_tokens
if prompt_tokens is None or completion_tokens is None or total_tokens is None:
return None
# ImageResponse may carry a default zeroed usage object even when provider
# usage metadata is absent. Treat this as missing usage and fall back.
if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0:
return None
input_tokens_details = getattr(usage, "input_tokens_details", None)
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
if input_tokens_details is not None:
prompt_tokens_details = PromptTokensDetailsWrapper(
text_tokens=getattr(input_tokens_details, "text_tokens", None),
image_tokens=getattr(input_tokens_details, "image_tokens", None),
cached_tokens=0,
)
normalized_usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
prompt_tokens_details=prompt_tokens_details,
completion_tokens_details=CompletionTokensDetailsWrapper(
text_tokens=0,
image_tokens=completion_tokens,
reasoning_tokens=0,
audio_tokens=0,
),
)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=normalized_usage,
custom_llm_provider=custom_llm_provider,
)
return prompt_cost + completion_cost
class CostCalculatorUtils:
@staticmethod
def _call_type_has_image_response(call_type: str) -> bool:
litellm/litellm_core_utils/prompt_templates/factory.py
+19 -8
View File
@@ -1766,6 +1766,7 @@ def convert_function_to_anthropic_tool_invoke(
def convert_to_anthropic_tool_invoke(
tool_calls: List[ChatCompletionAssistantToolCall],
web_search_results: Optional[List[Any]] = None,
tool_results: Optional[List[Any]] = None,
) -> List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]]:
"""
OpenAI tool invokes:
@@ -1840,12 +1841,18 @@ def convert_to_anthropic_tool_invoke(
}
anthropic_tool_invoke.append(_anthropic_server_tool_use)
# Add corresponding web_search_tool_result if available
# Add corresponding tool result if available.
# Check both web_search_results (web_search_tool_result / web_fetch_tool_result)
# and tool_results (bash_code_execution_tool_result, etc.)
_all_tool_results: List[Any] = []
if web_search_results:
for result in web_search_results:
if result.get("tool_use_id") == tool_id:
anthropic_tool_invoke.append(result)
break
_all_tool_results.extend(web_search_results)
if tool_results:
_all_tool_results.extend(tool_results)
for result in _all_tool_results:
if result.get("tool_use_id") == tool_id:
anthropic_tool_invoke.append(result)
break
else:
# Regular tool_use
sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id)
@@ -2472,9 +2479,10 @@ def anthropic_messages_pt( # noqa: PLR0915
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "server_tool_use":
assistant_content.append(m) # type: ignore
# handle tool_search_tool_result blocks
# handle all *_tool_result blocks (tool_search_tool_result,
# web_search_tool_result, bash_code_execution_tool_result, etc.)
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "tool_search_tool_result":
elif m.get("type", "").endswith("_tool_result"):
assistant_content.append(m) # type: ignore
elif (
"content" in assistant_content_block
@@ -2504,7 +2512,8 @@ def anthropic_messages_pt( # noqa: PLR0915
if (
assistant_tool_calls is not None
): # support assistant tool invoke conversion
# Get web_search_results from provider_specific_fields for server_tool_use reconstruction
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get(
"provider_specific_fields"
@@ -2517,9 +2526,11 @@ def anthropic_messages_pt( # noqa: PLR0915
_web_search_results = _provider_specific_fields.get(
"web_search_results"
)
_tool_results = _provider_specific_fields.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
tool_results=_tool_results,
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates
litellm/litellm_core_utils/realtime_streaming.py
+54 -30
View File
@@ -72,6 +72,9 @@ class RealTimeStreaming:
self.request_data: Dict = request_data or {}
# Violation counter for end_session_after_n_fails support
self._violation_count: int = 0
# When a text message is blocked, hold the guardrail reason so the next
# response.create can be rewritten to include the failure context.
self._pending_guardrail_message: Optional[str] = None
def _should_store_message(
self,
@@ -230,9 +233,9 @@ class RealTimeStreaming:
message, self.model, self.session_configuration_request
)
for msg in transformed:
await self.backend_ws.send(msg)
await self.backend_ws.send(msg) # type: ignore[union-attr, attr-defined]
else:
await self.backend_ws.send(message)
await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined]
def _has_realtime_guardrails(self) -> bool:
"""Return True if any callback is registered for realtime guardrail event types."""
@@ -261,18 +264,12 @@ class RealTimeStreaming:
When this returns True, we inject a session.update to disable the LLM's
auto-response so the guardrail can gate it first.
"""
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
return any(
isinstance(cb, CustomGuardrail)
and cb.should_run_guardrail(
data=self.request_data,
event_type=GuardrailEventHooks.realtime_input_transcription,
)
for cb in litellm.callbacks
)
Must match the same hook criteria as run_realtime_guardrails() so that
any guardrail that would actually check the transcript also disables
auto-response before the transcript arrives.
"""
return self._has_realtime_guardrails()
async def run_realtime_guardrails(
self,
@@ -335,18 +332,35 @@ class RealTimeStreaming:
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
# Return the error directly to the WebSocket consumer.
# Cancel any in-progress LLM response (e.g. VAD auto-response).
await self._send_to_backend(json.dumps({"type": "response.cancel"}))
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
{
"type": "error",
"error": {
"type": "guardrail_violation",
"message": error_msg,
"code": "content_policy_violation",
},
}
)
json.dumps({
"type": "error",
"error": {
"type": "guardrail_violation",
"message": error_msg,
"code": "content_policy_violation",
},
})
)
# Ask the LLM to voice the exact guardrail message so the
# user hears it as audio in voice sessions (not just text).
guardrail_prompt = (
f"Say exactly the following message to the user, word for word, "
f"do not add anything else: {error_msg}"
)
await self._send_to_backend(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": guardrail_prompt}],
},
}))
await self._send_to_backend(
json.dumps({"type": "response.create"})
)
self._violation_count += 1
@@ -362,7 +376,7 @@ class RealTimeStreaming:
"[realtime guardrail] ending session after violation %d",
self._violation_count,
)
await self.backend_ws.close()
await self.backend_ws.close() # type: ignore[union-attr, attr-defined]
verbose_logger.warning(
"[realtime guardrail] BLOCKED transcript (violation %d): %r",
@@ -502,11 +516,11 @@ class RealTimeStreaming:
try:
while True:
try:
raw_response = await self.backend_ws.recv(
raw_response = await self.backend_ws.recv( # type: ignore[union-attr]
decode=False
) # improves performance
except TypeError:
raw_response = await self.backend_ws.recv() # type: ignore[assignment]
raw_response = await self.backend_ws.recv() # type: ignore[union-attr, assignment]
if self.provider_config:
try:
@@ -559,7 +573,17 @@ class RealTimeStreaming:
combined_text
)
if blocked:
continue # don't forward to backend
# Store the guardrail reason so the next response.create
# (sent automatically by the client) is rewritten to
# include it as response instructions.
self._pending_guardrail_message = combined_text
continue # don't forward the original blocked message
if msg_type == "response.create" and self._pending_guardrail_message:
# The guardrail already sent the synthetic AI bubble — drop this
# response.create so OpenAI doesn't generate an additional response.
self._pending_guardrail_message = None
continue
except (json.JSONDecodeError, AttributeError):
pass
@@ -573,9 +597,9 @@ class RealTimeStreaming:
)
for msg in message:
await self.backend_ws.send(msg)
await self.backend_ws.send(msg) # type: ignore[union-attr]
else:
await self.backend_ws.send(message)
await self.backend_ws.send(message) # type: ignore[union-attr]
except Exception as e:
verbose_logger.debug(f"Error in client ack messages: {e}")
litellm/litellm_core_utils/streaming_handler.py
+92 -60
View File
@@ -13,6 +13,7 @@ from typing import (
Dict,
Iterator,
List,
NoReturn,
Optional,
Union,
cast,
@@ -96,6 +97,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._stream_created_time: float = time.time()
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams(
**self.logging_obj.model_call_details.get("litellm_params", {})
@@ -161,6 +163,20 @@ class CustomStreamWrapper:
self.is_function_call = self.check_is_function_call(logging_obj=logging_obj)
self.created: Optional[int] = None
def _check_max_streaming_duration(self) -> None:
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
from litellm.constants import LITELLM_MAX_STREAMING_DURATION_SECONDS
if LITELLM_MAX_STREAMING_DURATION_SECONDS is None:
return
elapsed = time.time() - self._stream_created_time
if elapsed > LITELLM_MAX_STREAMING_DURATION_SECONDS:
raise litellm.Timeout(
message=f"Stream exceeded max streaming duration of {LITELLM_MAX_STREAMING_DURATION_SECONDS}s (elapsed {elapsed:.1f}s)",
model=self.model or "",
llm_provider=self.custom_llm_provider or "",
)
def __iter__(self) -> Iterator["ModelResponseStream"]:
return self
@@ -1236,27 +1252,27 @@ class CustomStreamWrapper:
else:
completion_obj["content"] = str(chunk)
elif self.custom_llm_provider == "petals":
if len(self.completion_stream) == 0:
if self.completion_stream is None or len(self.completion_stream) == 0:
if self.received_finish_reason is not None:
raise StopIteration
else:
self.received_finish_reason = "stop"
chunk_size = 30
new_chunk = self.completion_stream[:chunk_size]
new_chunk = self.completion_stream[:chunk_size] # type: ignore[index]
completion_obj["content"] = new_chunk
self.completion_stream = self.completion_stream[chunk_size:]
self.completion_stream = self.completion_stream[chunk_size:] # type: ignore[index]
elif self.custom_llm_provider == "palm":
# fake streaming
response_obj = {}
if len(self.completion_stream) == 0:
if self.completion_stream is None or len(self.completion_stream) == 0:
if self.received_finish_reason is not None:
raise StopIteration
else:
self.received_finish_reason = "stop"
chunk_size = 30
new_chunk = self.completion_stream[:chunk_size]
new_chunk = self.completion_stream[:chunk_size] # type: ignore[index]
completion_obj["content"] = new_chunk
self.completion_stream = self.completion_stream[chunk_size:]
self.completion_stream = self.completion_stream[chunk_size:] # type: ignore[index]
elif self.custom_llm_provider == "triton":
response_obj = self.handle_triton_stream(chunk)
completion_obj["content"] = response_obj["text"]
@@ -1743,6 +1759,7 @@ class CustomStreamWrapper:
and self.custom_llm_provider == "cached_response"
):
cache_hit = True
self._check_max_streaming_duration()
try:
if self.completion_stream is None:
self.fetch_sync_stream()
@@ -1755,7 +1772,7 @@ class CustomStreamWrapper:
):
chunk = self.completion_stream
else:
chunk = next(self.completion_stream)
chunk = next(self.completion_stream) # type: ignore[arg-type]
if chunk is not None and chunk != b"":
print_verbose(
f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk.decode('utf-8', errors='replace') if isinstance(chunk, bytes) else chunk}; custom_llm_provider: {self.custom_llm_provider}"
@@ -1883,14 +1900,7 @@ class CustomStreamWrapper:
threading.Thread(
target=self.logging_obj.failure_handler, args=(e, traceback_exception)
).start()
if isinstance(e, OpenAIError):
raise e
else:
raise exception_type(
model=self.model,
original_exception=e,
custom_llm_provider=self.custom_llm_provider,
)
self._handle_stream_fallback_error(e)
def fetch_sync_stream(self):
if self.completion_stream is None and self.make_call is not None:
@@ -1917,12 +1927,13 @@ class CustomStreamWrapper:
and self.custom_llm_provider == "cached_response"
):
cache_hit = True
self._check_max_streaming_duration()
try:
if self.completion_stream is None:
await self.fetch_stream()
if is_async_iterable(self.completion_stream):
async for chunk in self.completion_stream:
async for chunk in self.completion_stream: # type: ignore[union-attr]
if chunk == "None" or chunk is None:
continue # skip None chunks
@@ -1951,22 +1962,24 @@ class CustomStreamWrapper:
self.rules.post_call_rules(
input=self.response_uptil_now, model=self.model
)
# Store a shallow copy so usage stripping below
# does not mutate the stored chunk.
self.chunks.append(processed_chunk.model_copy())
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk:
processed_chunk = self._add_mcp_list_tools_to_first_chunk(processed_chunk)
self.sent_first_chunk = True
if (
_has_usage = (
hasattr(processed_chunk, "usage")
and getattr(processed_chunk, "usage", None) is not None
):
)
if _has_usage:
# Store a copy ONLY when usage stripping below will mutate
# the chunk. For non-usage chunks (vast majority), store
# directly to avoid expensive model_copy() per chunk.
self.chunks.append(processed_chunk.model_copy())
# Strip usage from the outgoing chunk so it's not sent twice
# (once in the chunk, once in _hidden_params).
# Create a new object without usage, matching sync behavior.
# The copy in self.chunks retains usage for calculate_total_usage().
obj_dict = processed_chunk.model_dump()
if "usage" in obj_dict:
del obj_dict["usage"]
@@ -1978,6 +1991,9 @@ class CustomStreamWrapper:
)
if is_empty:
continue
else:
# No usage data — safe to store directly without copying
self.chunks.append(processed_chunk)
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
@@ -2004,7 +2020,7 @@ class CustomStreamWrapper:
):
chunk = self.completion_stream
else:
chunk = next(self.completion_stream)
chunk = next(self.completion_stream) # type: ignore[arg-type]
if chunk is not None and chunk != b"":
processed_chunk = self.chunk_creator(chunk=chunk)
if processed_chunk is None:
@@ -2102,7 +2118,25 @@ class CustomStreamWrapper:
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception) # type: ignore
)
## Map to OpenAI Exception
self._handle_stream_fallback_error(e)
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
Maps the raw exception to an OpenAI-compatible type, then decides
whether to raise it directly (non-retriable 4xx) or wrap it in
MidStreamFallbackError so the Router can trigger a fallback.
429 (rate-limit) is explicitly exempted from the 4xx filter because
it is transient and the Router should switch to another model group.
"""
from litellm.exceptions import MidStreamFallbackError
# Map to OpenAI exception format
if isinstance(e, OpenAIError):
mapped_exception: Exception = e
else:
try:
mapped_exception = exception_type(
model=self.model,
@@ -2114,46 +2148,44 @@ class CustomStreamWrapper:
except Exception as mapping_error:
mapped_exception = mapping_error
def _normalize_status_code(exc: Exception) -> Optional[int]:
"""
Best-effort status_code extraction.
Uses status_code on the exception, then falls back to the response.
"""
def _normalize_status_code(exc: Exception) -> Optional[int]:
"""Best-effort status_code extraction."""
try:
code = getattr(exc, "status_code", None)
if code is not None:
return int(code)
except Exception:
pass
response = getattr(exc, "response", None)
if response is not None:
try:
code = getattr(exc, "status_code", None)
if code is not None:
return int(code)
status_code = getattr(response, "status_code", None)
if status_code is not None:
return int(status_code)
except Exception:
pass
return None
response = getattr(exc, "response", None)
if response is not None:
try:
status_code = getattr(response, "status_code", None)
if status_code is not None:
return int(status_code)
except Exception:
pass
return None
mapped_status_code = _normalize_status_code(mapped_exception)
original_status_code = _normalize_status_code(e)
mapped_status_code = _normalize_status_code(mapped_exception)
original_status_code = _normalize_status_code(e)
# Raise non-retriable client errors directly (skip fallback).
# Exception: 429 (rate-limit) IS retriable/transient — allow it
# through so the Router can switch to a different model group.
if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429:
raise mapped_exception
if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429:
raise mapped_exception
if mapped_status_code is not None and 400 <= mapped_status_code < 500:
raise mapped_exception
if original_status_code is not None and 400 <= original_status_code < 500:
raise mapped_exception
from litellm.exceptions import MidStreamFallbackError
raise MidStreamFallbackError(
message=str(mapped_exception),
model=self.model,
llm_provider=self.custom_llm_provider or "anthropic",
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
)
raise MidStreamFallbackError(
message=str(mapped_exception),
model=self.model,
llm_provider=self.custom_llm_provider or "anthropic",
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
)
@staticmethod
def _strip_sse_data_from_chunk(chunk: Optional[str]) -> Optional[str]:
litellm/llms/anthropic/chat/transformation.py
+11 -5
View File
@@ -997,12 +997,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
reasoning_effort=value, model=model
)
# For Claude 4.6 models, effort is controlled via output_config,
# not thinking budget_tokens. Map reasoning_effort to output_config.
if AnthropicConfig._is_claude_4_6_model(model):
# Map reasoning_effort to Anthropic's output_config for 4.6 models
effort_map = {"minimal": "low", "low": "low", "medium": "medium", "high": "high", "max": "max"}
anthropic_effort = effort_map.get(value)
if anthropic_effort is not None:
optional_params["output_config"] = {"effort": anthropic_effort}
effort_map = {
"low": "low",
"minimal": "low",
"medium": "medium",
"high": "high",
"max": "max",
}
mapped_effort = effort_map.get(value, value)
optional_params["output_config"] = {"effort": mapped_effort}
elif param == "web_search_options" and isinstance(value, dict):
hosted_web_search_tool = self.map_web_search_tool(
cast(OpenAIWebSearchOptions, value)
litellm/llms/anthropic/common_utils.py
+15 -2
View File
@@ -31,6 +31,15 @@ def is_anthropic_oauth_key(value: Optional[str]) -> bool:
value = value[7:]
return value.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)
def _merge_beta_headers(existing: Optional[str], new_beta: str) -> str:
"""Merge a new beta value into an existing comma-separated anthropic-beta header."""
if not existing:
return new_beta
betas = {b.strip() for b in existing.split(",") if b.strip()}
betas.add(new_beta)
return ",".join(sorted(betas))
def optionally_handle_anthropic_oauth(
headers: dict, api_key: Optional[str]
) -> tuple[dict, Optional[str]]:
@@ -52,14 +61,18 @@ def optionally_handle_anthropic_oauth(
if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.replace("Bearer ", "")
headers.pop("x-api-key", None)
headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
headers["anthropic-beta"] = _merge_beta_headers(
headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER
)
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
# Check api_key directly (standard chat/completion flow)
if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX):
headers.pop("x-api-key", None)
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
headers["anthropic-beta"] = _merge_beta_headers(
headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER
)
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
litellm/llms/anthropic/count_tokens/transformation.py
+9 -1
View File
@@ -63,12 +63,20 @@ class AnthropicCountTokensConfig:
Returns:
Dictionary of required headers
"""
return {
from litellm.llms.anthropic.common_utils import (
optionally_handle_anthropic_oauth,
)
headers: Dict[str, str] = {
"Content-Type": "application/json",
"x-api-key": api_key,
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
}
headers, _ = optionally_handle_anthropic_oauth(
headers=headers, api_key=api_key
)
return headers
def validate_request(
self, model: str, messages: List[Dict[str, Any]]
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
+43 -22
View File
@@ -25,8 +25,24 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager, client
from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
from .utils import AnthropicMessagesRequestUtils, mock_response
# Providers that are routed directly to the OpenAI Responses API instead of
# going through chat/completions.
_RESPONSES_API_PROVIDERS = frozenset({"openai"})
def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool:
"""Return True when the provider should use the Responses API path.
Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
opt out and route OpenAI/Azure requests through chat/completions instead.
"""
if litellm.use_chat_completions_url_for_anthropic_messages:
return False
return custom_llm_provider in _RESPONSES_API_PROVIDERS
####### ENVIRONMENT VARIABLES ###################
# Initialize any necessary instances or variables here
base_llm_http_handler = BaseLLMHTTPHandler()
@@ -282,29 +298,34 @@ def anthropic_messages_handler(
)
)
if anthropic_messages_provider_config is None:
# Handle non-Anthropic models using the adapter
return (
LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
_is_async=is_async,
api_key=api_key,
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
**kwargs,
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
_shared_kwargs = dict(
max_tokens=max_tokens,
messages=messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
_is_async=is_async,
api_key=api_key,
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
if _should_route_to_responses_api(custom_llm_provider):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
**_shared_kwargs
)
return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
**_shared_kwargs
)
if custom_llm_provider is None:
litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"]
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
+229
View File
@@ -0,0 +1,229 @@
"""
Handler for the Anthropic v1/messages -> OpenAI Responses API path.
Used when the target model is an OpenAI or Azure model.
"""
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
import litellm
from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter()
def _build_responses_kwargs(
*,
max_tokens: int,
messages: List[Dict],
model: str,
context_management: Optional[Dict] = None,
metadata: Optional[Dict] = None,
output_config: Optional[Dict] = None,
stop_sequences: Optional[List[str]] = None,
stream: Optional[bool] = False,
system: Optional[str] = None,
temperature: Optional[float] = None,
thinking: Optional[Dict] = None,
tool_choice: Optional[Dict] = None,
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
output_format: Optional[Dict] = None,
extra_kwargs: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
"""
# Build a typed AnthropicMessagesRequest for the adapter
request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens}
if context_management:
request_data["context_management"] = context_management
if output_config:
request_data["output_config"] = output_config
if metadata:
request_data["metadata"] = metadata
if system:
request_data["system"] = system
if temperature is not None:
request_data["temperature"] = temperature
if thinking:
request_data["thinking"] = thinking
if tool_choice:
request_data["tool_choice"] = tool_choice
if tools:
request_data["tools"] = tools
if top_p is not None:
request_data["top_p"] = top_p
if output_format:
request_data["output_format"] = output_format
anthropic_request = AnthropicMessagesRequest(**request_data) # type: ignore[typeddict-item]
responses_kwargs = _ADAPTER.translate_request(anthropic_request)
if stream:
responses_kwargs["stream"] = True
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
excluded = {"anthropic_messages"}
for key, value in (extra_kwargs or {}).items():
if key == "litellm_logging_obj" and value is not None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
)
from litellm.types.utils import CallTypes
if isinstance(value, LiteLLMLoggingObject):
# Reclassify as acompletion so the success handler doesn't try to
# validate the Responses API event as an AnthropicResponse.
# (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.)
setattr(value, "call_type", CallTypes.acompletion.value)
responses_kwargs[key] = value
elif key not in excluded and key not in responses_kwargs and value is not None:
responses_kwargs[key] = value
return responses_kwargs
class LiteLLMMessagesToResponsesAPIHandler:
"""
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
calling litellm.responses() / litellm.aresponses() directly and translating
the response back to Anthropic format.
"""
@staticmethod
async def async_anthropic_messages_handler(
max_tokens: int,
messages: List[Dict],
model: str,
context_management: Optional[Dict] = None,
metadata: Optional[Dict] = None,
output_config: Optional[Dict] = None,
stop_sequences: Optional[List[str]] = None,
stream: Optional[bool] = False,
system: Optional[str] = None,
temperature: Optional[float] = None,
thinking: Optional[Dict] = None,
tool_choice: Optional[Dict] = None,
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
output_format: Optional[Dict] = None,
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
responses_kwargs = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
result = await litellm.aresponses(**responses_kwargs)
if stream:
wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
@staticmethod
def anthropic_messages_handler(
max_tokens: int,
messages: List[Dict],
model: str,
context_management: Optional[Dict] = None,
metadata: Optional[Dict] = None,
output_config: Optional[Dict] = None,
stop_sequences: Optional[List[str]] = None,
stream: Optional[bool] = False,
system: Optional[str] = None,
temperature: Optional[float] = None,
thinking: Optional[Dict] = None,
tool_choice: Optional[Dict] = None,
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
output_format: Optional[Dict] = None,
_is_async: bool = False,
**kwargs,
) -> Union[
AnthropicMessagesResponse,
AsyncIterator[Any],
Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]],
]:
if _is_async:
return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
**kwargs,
)
# Sync path
responses_kwargs = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
result = litellm.responses(**responses_kwargs)
if stream:
wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
+265
View File
@@ -0,0 +1,265 @@
# What is this?
## Translates OpenAI call to Anthropic `/v1/messages` format
import json
import traceback
from collections import deque
from typing import Any, AsyncIterator, Dict
from litellm import verbose_logger
from litellm._uuid import uuid
class AnthropicResponsesStreamWrapper:
"""
Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format.
Responses API event flow (relevant subset):
response.created -> message_start
response.output_item.added -> content_block_start (if message/function_call)
response.output_text.delta -> content_block_delta (text_delta)
response.reasoning_summary_text.delta -> content_block_delta (thinking_delta)
response.function_call_arguments.delta -> content_block_delta (input_json_delta)
response.output_item.done -> content_block_stop
response.completed -> message_delta + message_stop
"""
def __init__(
self,
responses_stream: Any,
model: str,
) -> None:
self.responses_stream = responses_stream
self.model = model
self._message_id: str = f"msg_{uuid.uuid4()}"
self._current_block_index: int = -1
# Map item_id -> content_block_index so we can stop the right block later
self._item_id_to_block_index: Dict[str, int] = {}
# Track open function_call items by item_id so we can emit tool_use start
self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator
self._sent_message_start = False
self._sent_message_stop = False
self._chunk_queue: deque = deque()
def _make_message_start(self) -> Dict[str, Any]:
return {
"type": "message_start",
"message": {
"id": self._message_id,
"type": "message",
"role": "assistant",
"content": [],
"model": self.model,
"stop_reason": None,
"stop_sequence": None,
"usage": {
"input_tokens": 0,
"output_tokens": 0,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
},
},
}
def _next_block_index(self) -> int:
self._current_block_index += 1
return self._current_block_index
def _process_event(self, event: Any) -> None: # noqa: PLR0915
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
event_type = getattr(event, "type", None)
if event_type is None and isinstance(event, dict):
event_type = event.get("type")
if event_type is None:
return
# ---- message_start ----
if event_type == "response.created":
self._sent_message_start = True
self._chunk_queue.append(self._make_message_start())
return
# ---- content_block_start for a new output message item ----
if event_type == "response.output_item.added":
item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None)
if item is None:
return
item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None)
item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None)
if item_type == "message":
block_idx = self._next_block_index()
if item_id:
self._item_id_to_block_index[item_id] = block_idx
self._chunk_queue.append({
"type": "content_block_start",
"index": block_idx,
"content_block": {"type": "text", "text": ""},
})
elif item_type == "function_call":
call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or ""
name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or ""
block_idx = self._next_block_index()
if item_id:
self._item_id_to_block_index[item_id] = block_idx
self._pending_tool_ids[item_id] = call_id
self._chunk_queue.append({
"type": "content_block_start",
"index": block_idx,
"content_block": {
"type": "tool_use",
"id": call_id,
"name": name,
"input": {},
},
})
elif item_type == "reasoning":
block_idx = self._next_block_index()
if item_id:
self._item_id_to_block_index[item_id] = block_idx
self._chunk_queue.append({
"type": "content_block_start",
"index": block_idx,
"content_block": {"type": "thinking", "thinking": ""},
})
return
# ---- text delta ----
if event_type == "response.output_text.delta":
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
self._chunk_queue.append({
"type": "content_block_delta",
"index": block_idx,
"delta": {"type": "text_delta", "text": delta},
})
return
# ---- reasoning summary text delta ----
if event_type == "response.reasoning_summary_text.delta":
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
self._chunk_queue.append({
"type": "content_block_delta",
"index": block_idx,
"delta": {"type": "thinking_delta", "thinking": delta},
})
return
# ---- function call arguments delta ----
if event_type == "response.function_call_arguments.delta":
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
self._chunk_queue.append({
"type": "content_block_delta",
"index": block_idx,
"delta": {"type": "input_json_delta", "partial_json": delta},
})
return
# ---- output item done -> content_block_stop ----
if event_type == "response.output_item.done":
item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None)
item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
self._chunk_queue.append({
"type": "content_block_stop",
"index": block_idx,
})
return
# ---- response completed -> message_delta + message_stop ----
if event_type in ("response.completed", "response.failed", "response.incomplete"):
response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None)
stop_reason = "end_turn"
input_tokens = 0
output_tokens = 0
cache_creation_tokens = 0
cache_read_tokens = 0
if response_obj is not None:
status = getattr(response_obj, "status", None)
if status == "incomplete":
stop_reason = "max_tokens"
usage = getattr(response_obj, "usage", None)
if usage is not None:
input_tokens = getattr(usage, "input_tokens", 0) or 0
output_tokens = getattr(usage, "output_tokens", 0) or 0
cache_creation_tokens = getattr(usage, "input_tokens_details", None) # type: ignore[assignment]
cache_read_tokens = getattr(usage, "output_tokens_details", None) # type: ignore[assignment]
# Prefer direct cache fields if present
cache_creation_tokens = int(getattr(usage, "cache_creation_input_tokens", 0) or 0)
cache_read_tokens = int(getattr(usage, "cache_read_input_tokens", 0) or 0)
# Check if tool_use was in the output to override stop_reason
if response_obj is not None:
output = getattr(response_obj, "output", []) or []
for out_item in output:
out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None)
if out_type == "function_call":
stop_reason = "tool_use"
break
usage_delta: Dict[str, Any] = {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
}
if cache_creation_tokens:
usage_delta["cache_creation_input_tokens"] = cache_creation_tokens
if cache_read_tokens:
usage_delta["cache_read_input_tokens"] = cache_read_tokens
self._chunk_queue.append({
"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
"usage": usage_delta,
})
self._chunk_queue.append({"type": "message_stop"})
self._sent_message_stop = True
return
def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
return self
async def __anext__(self) -> Dict[str, Any]:
# Return any queued chunks first
if self._chunk_queue:
return self._chunk_queue.popleft()
# Emit message_start if not yet done (fallback if response.created wasn't fired)
if not self._sent_message_start:
self._sent_message_start = True
self._chunk_queue.append(self._make_message_start())
return self._chunk_queue.popleft()
# Consume the upstream stream
try:
async for event in self.responses_stream:
self._process_event(event)
if self._chunk_queue:
return self._chunk_queue.popleft()
except StopAsyncIteration:
pass
except Exception as e:
verbose_logger.error(
f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}"
)
# Drain any remaining queued chunks
if self._chunk_queue:
return self._chunk_queue.popleft()
raise StopAsyncIteration
async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
"""Yield SSE-encoded bytes for each Anthropic event chunk."""
async for chunk in self:
if isinstance(chunk, dict):
event_type: str = str(chunk.get("type", "message"))
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
yield payload.encode()
else:
yield chunk
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
+450
View File
@@ -0,0 +1,450 @@
"""
Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API.
This module owns all format conversions for the direct v1/messages -> Responses API
path used for OpenAI and Azure models.
"""
import json
from typing import Any, Dict, List, Optional, Union, cast
from litellm.types.llms.anthropic import (
AllAnthropicToolsValues,
AnthopicMessagesAssistantMessageParam,
AnthropicFinishReason,
AnthropicMessagesRequest,
AnthropicMessagesToolChoice,
AnthropicMessagesUserMessageParam,
AnthropicResponseContentBlockText,
AnthropicResponseContentBlockThinking,
AnthropicResponseContentBlockToolUse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
AnthropicUsage,
)
from litellm.types.llms.openai import ResponsesAPIResponse
class LiteLLMAnthropicToResponsesAPIAdapter:
"""
Converts Anthropic /v1/messages requests to OpenAI Responses API format and
converts Responses API responses back to Anthropic format.
"""
# ------------------------------------------------------------------ #
# Request translation: Anthropic -> Responses API #
# ------------------------------------------------------------------ #
@staticmethod
def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]:
"""Convert Anthropic image source to a URL string."""
source_type = source.get("type")
if source_type == "base64":
media_type = source.get("media_type", "image/jpeg")
data = source.get("data", "")
return f"data:{media_type};base64,{data}" if data else None
elif source_type == "url":
return source.get("url")
return None
def translate_messages_to_responses_input( # noqa: PLR0915
self,
messages: List[
Union[
AnthropicMessagesUserMessageParam,
AnthopicMessagesAssistantMessageParam,
]
],
) -> List[Dict[str, Any]]:
"""
Convert Anthropic messages list to Responses API `input` items.
Mapping:
user text -> message(role=user, input_text)
user image -> message(role=user, input_image)
user tool_result -> function_call_output
assistant text -> message(role=assistant, output_text)
assistant tool_use -> function_call
"""
input_items: List[Dict[str, Any]] = []
for m in messages:
role = m["role"]
content = m.get("content")
if role == "user":
if isinstance(content, str):
input_items.append({
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": content}],
})
elif isinstance(content, list):
user_parts: List[Dict[str, Any]] = []
for block in content:
if not isinstance(block, dict):
continue
btype = block.get("type")
if btype == "text":
user_parts.append({"type": "input_text", "text": block.get("text", "")})
elif btype == "image":
url = self._translate_anthropic_image_source_to_url(block.get("source", {}))
if url:
user_parts.append({"type": "input_image", "image_url": url})
elif btype == "tool_result":
tool_use_id = block.get("tool_use_id", "")
inner = block.get("content")
if inner is None:
output_text = ""
elif isinstance(inner, str):
output_text = inner
elif isinstance(inner, list):
parts = [
c.get("text", "")
for c in inner
if isinstance(c, dict) and c.get("type") == "text"
]
output_text = "\n".join(parts)
else:
output_text = str(inner)
# tool_result is a top-level item, not inside the message
input_items.append({
"type": "function_call_output",
"call_id": tool_use_id,
"output": output_text,
})
if user_parts:
input_items.append({
"type": "message",
"role": "user",
"content": user_parts,
})
elif role == "assistant":
if isinstance(content, str):
input_items.append({
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": content}],
})
elif isinstance(content, list):
asst_parts: List[Dict[str, Any]] = []
for block in content:
if not isinstance(block, dict):
continue
btype = block.get("type")
if btype == "text":
asst_parts.append({"type": "output_text", "text": block.get("text", "")})
elif btype == "tool_use":
# tool_use becomes a top-level function_call item
input_items.append({
"type": "function_call",
"call_id": block.get("id", ""),
"name": block.get("name", ""),
"arguments": json.dumps(block.get("input", {})),
})
elif btype == "thinking":
thinking_text = block.get("thinking", "")
if thinking_text:
asst_parts.append({"type": "output_text", "text": thinking_text})
if asst_parts:
input_items.append({
"type": "message",
"role": "assistant",
"content": asst_parts,
})
return input_items
def translate_tools_to_responses_api(
self,
tools: List[AllAnthropicToolsValues],
) -> List[Dict[str, Any]]:
"""Convert Anthropic tool definitions to Responses API function tools."""
result: List[Dict[str, Any]] = []
for tool in tools:
tool_dict = cast(Dict[str, Any], tool)
tool_type = tool_dict.get("type", "")
tool_name = tool_dict.get("name", "")
# web_search tool
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
result.append({"type": "web_search_preview"})
continue
func_tool: Dict[str, Any] = {"type": "function", "name": tool_name}
if "description" in tool_dict:
func_tool["description"] = tool_dict["description"]
if "input_schema" in tool_dict:
func_tool["parameters"] = tool_dict["input_schema"]
result.append(func_tool)
return result
@staticmethod
def translate_tool_choice_to_responses_api(
tool_choice: AnthropicMessagesToolChoice,
) -> Dict[str, Any]:
"""Convert Anthropic tool_choice to Responses API tool_choice."""
tc_type = tool_choice.get("type")
if tc_type == "any":
return {"type": "required"}
elif tc_type == "tool":
return {"type": "function", "name": tool_choice.get("name", "")}
return {"type": "auto"}
@staticmethod
def translate_context_management_to_responses_api(
context_management: Dict[str, Any],
) -> Optional[List[Dict[str, Any]]]:
"""
Convert Anthropic context_management dict to OpenAI Responses API array format.
Anthropic format: {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]}
OpenAI format: [{"type": "compaction", "compact_threshold": 150000}]
"""
if not isinstance(context_management, dict):
return None
edits = context_management.get("edits", [])
if not isinstance(edits, list):
return None
result: List[Dict[str, Any]] = []
for edit in edits:
if not isinstance(edit, dict):
continue
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
entry: Dict[str, Any] = {"type": "compaction"}
trigger = edit.get("trigger")
if isinstance(trigger, dict) and trigger.get("value") is not None:
entry["compact_threshold"] = int(trigger["value"])
result.append(entry)
return result if result else None
@staticmethod
def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""
Convert Anthropic thinking param to Responses API reasoning param.
thinking.budget_tokens maps to reasoning effort:
>= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal
"""
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
return None
budget = thinking.get("budget_tokens", 0)
if budget >= 10000:
effort = "high"
elif budget >= 5000:
effort = "medium"
elif budget >= 2000:
effort = "low"
else:
effort = "minimal"
return {"effort": effort, "summary": "detailed"}
def translate_request(
self,
anthropic_request: AnthropicMessagesRequest,
) -> Dict[str, Any]:
"""
Translate a full Anthropic /v1/messages request dict to
litellm.responses() / litellm.aresponses() kwargs.
"""
model: str = anthropic_request["model"]
messages_list = cast(
List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]],
anthropic_request["messages"],
)
responses_kwargs: Dict[str, Any] = {
"model": model,
"input": self.translate_messages_to_responses_input(messages_list),
}
# system -> instructions
system = anthropic_request.get("system")
if system:
if isinstance(system, str):
responses_kwargs["instructions"] = system
elif isinstance(system, list):
text_parts = [
b.get("text", "")
for b in system
if isinstance(b, dict) and b.get("type") == "text"
]
responses_kwargs["instructions"] = "\n".join(filter(None, text_parts))
# max_tokens -> max_output_tokens
max_tokens = anthropic_request.get("max_tokens")
if max_tokens:
responses_kwargs["max_output_tokens"] = max_tokens
# temperature / top_p passed through
if "temperature" in anthropic_request:
responses_kwargs["temperature"] = anthropic_request["temperature"]
if "top_p" in anthropic_request:
responses_kwargs["top_p"] = anthropic_request["top_p"]
# tools
tools = anthropic_request.get("tools")
if tools:
responses_kwargs["tools"] = self.translate_tools_to_responses_api(
cast(List[AllAnthropicToolsValues], tools)
)
# tool_choice
tool_choice = anthropic_request.get("tool_choice")
if tool_choice:
responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api(
cast(AnthropicMessagesToolChoice, tool_choice)
)
# thinking -> reasoning
thinking = anthropic_request.get("thinking")
if isinstance(thinking, dict):
reasoning = self.translate_thinking_to_reasoning(thinking)
if reasoning:
responses_kwargs["reasoning"] = reasoning
# output_format / output_config.format -> text format
# output_format: {"type": "json_schema", "schema": {...}}
# output_config: {"format": {"type": "json_schema", "schema": {...}}}
output_format: Any = anthropic_request.get("output_format")
output_config = anthropic_request.get("output_config")
if not isinstance(output_format, dict) and isinstance(output_config, dict):
output_format = output_config.get("format") # type: ignore[assignment]
if isinstance(output_format, dict) and output_format.get("type") == "json_schema":
schema = output_format.get("schema")
if schema:
responses_kwargs["text"] = {
"format": {
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": True,
}
}
# context_management: Anthropic dict -> OpenAI array
context_management = anthropic_request.get("context_management")
if isinstance(context_management, dict):
openai_cm = self.translate_context_management_to_responses_api(context_management)
if openai_cm is not None:
responses_kwargs["context_management"] = openai_cm
# metadata user_id -> user
metadata = anthropic_request.get("metadata")
if isinstance(metadata, dict) and "user_id" in metadata:
responses_kwargs["user"] = str(metadata["user_id"])[:64]
return responses_kwargs
# ------------------------------------------------------------------ #
# Response translation: Responses API -> Anthropic #
# ------------------------------------------------------------------ #
def translate_response(
self,
response: ResponsesAPIResponse,
) -> AnthropicMessagesResponse:
"""
Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse.
"""
from openai.types.responses import (
ResponseFunctionToolCall,
ResponseOutputMessage,
ResponseReasoningItem,
)
from litellm.types.llms.openai import ResponseAPIUsage
content: List[Dict[str, Any]] = []
stop_reason: AnthropicFinishReason = "end_turn"
for item in response.output:
if isinstance(item, ResponseReasoningItem):
for summary in item.summary:
text = getattr(summary, "text", "")
if text:
content.append(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
)
elif isinstance(item, ResponseOutputMessage):
for part in item.content:
if getattr(part, "type", None) == "output_text":
content.append(
AnthropicResponseContentBlockText(
type="text", text=getattr(part, "text", "")
).model_dump()
)
elif isinstance(item, ResponseFunctionToolCall):
try:
input_data = json.loads(item.arguments) if item.arguments else {}
except (json.JSONDecodeError, TypeError):
input_data = {}
content.append(
AnthropicResponseContentBlockToolUse(
type="tool_use",
id=item.call_id or item.id or "",
name=item.name,
input=input_data,
).model_dump()
)
stop_reason = "tool_use"
elif isinstance(item, dict):
item_type = item.get("type")
if item_type == "message":
for part in item.get("content", []):
if isinstance(part, dict) and part.get("type") == "output_text":
content.append(
AnthropicResponseContentBlockText(
type="text", text=part.get("text", "")
).model_dump()
)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))
except (json.JSONDecodeError, TypeError):
input_data = {}
content.append(
AnthropicResponseContentBlockToolUse(
type="tool_use",
id=item.get("call_id") or item.get("id", ""),
name=item.get("name", ""),
input=input_data,
).model_dump()
)
stop_reason = "tool_use"
# status -> stop_reason override
if response.status == "incomplete":
stop_reason = "max_tokens"
# usage
raw_usage: Optional[ResponseAPIUsage] = response.usage
input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0)
output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0)
anthropic_usage = AnthropicUsage(
input_tokens=input_tokens,
output_tokens=output_tokens,
)
return AnthropicMessagesResponse(
id=response.id,
type="message",
role="assistant",
model=response.model or "unknown-model",
stop_sequence=None,
usage=anthropic_usage, # type: ignore
content=content, # type: ignore
stop_reason=stop_reason,
)
litellm/llms/azure/azure.py
+20 -1
View File
@@ -343,6 +343,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers, response = self.make_sync_azure_openai_chat_completion_request(
azure_client=azure_client, data=data, timeout=timeout
)
if isinstance(response, str):
raise AzureOpenAIError(
status_code=500,
message=f"Unexpected string response from Azure: {response[:500]}",
)
stringified_response = response.model_dump()
## LOGGING
logging_obj.post_call(
@@ -432,6 +437,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
)
logging_obj.model_call_details["response_headers"] = headers
if isinstance(response, str):
raise AzureOpenAIError(
status_code=500,
message=f"Unexpected string response from Azure: {response[:500]}",
)
stringified_response = response.model_dump()
logging_obj.post_call(
input=data["messages"],
@@ -690,7 +700,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
status_code=raw_response.status_code or 500,
message=f"Failed to parse raw Azure embedding response: {str(json_error)}"
) from json_error
if isinstance(response, str):
raise AzureOpenAIError(
status_code=raw_response.status_code or 500,
message=f"Unexpected string response from Azure: {response[:500]}",
)
stringified_response = response.model_dump()
## LOGGING
@@ -792,6 +806,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raw_response = azure_client.embeddings.with_raw_response.create(**data, timeout=timeout) # type: ignore
headers = dict(raw_response.headers)
response = raw_response.parse()
if isinstance(response, str):
raise AzureOpenAIError(
status_code=raw_response.status_code or 500,
message=f"Unexpected string response from Azure: {response[:500]}",
)
## LOGGING
logging_obj.post_call(
input=input,
litellm/llms/azure/realtime/handler.py
+5 -4
View File
@@ -33,7 +33,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
self,
api_base: str,
model: str,
api_version: str,
api_version: Optional[str],
realtime_protocol: Optional[str] = None,
) -> str:
"""
@@ -56,8 +56,9 @@ class AzureOpenAIRealtime(AzureChatCompletion):
"""
api_base = api_base.replace("https://", "wss://")
# Determine path based on realtime_protocol
if realtime_protocol in ("GA", "v1"):
# Determine path based on realtime_protocol (case-insensitive)
_is_ga = realtime_protocol is not None and realtime_protocol.upper() in ("GA", "V1")
if _is_ga:
path = "/openai/v1/realtime"
return f"{api_base}{path}?model={model}"
else:
@@ -85,7 +86,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
if api_base is None:
raise ValueError("api_base is required for Azure OpenAI calls")
if api_version is None:
if api_version is None and (realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")):
raise ValueError("api_version is required for Azure OpenAI calls")
url = self._construct_url(
litellm/llms/base_llm/ocr/transformation.py
+8 -2
View File
@@ -15,7 +15,9 @@ else:
LiteLLMLoggingObj = Any
# DocumentType for OCR - Mistral format document dict
# DocumentType for OCR - providers always receive a dict with
# type="document_url" or type="image_url" (str values only).
# File-type inputs are preprocessed to this format in litellm/ocr/main.py.
DocumentType = Dict[str, str]
@@ -141,9 +143,13 @@ class BaseOCRConfig:
Transform OCR request to provider-specific format.
Override in provider-specific implementations.
Note: By the time this method is called, any file-type documents have already
been converted to document_url/image_url format with base64 data URIs by
the preprocessing in litellm/ocr/main.py.
Args:
model: Model name
document: Document to process (Mistral format dict, or file path, bytes, etc.)
document: Document to process - always a dict with type="document_url" or type="image_url"
optional_params: Optional parameters for the request
headers: Request headers
litellm/llms/bedrock/chat/converse_handler.py
+17 -5
View File
@@ -4,6 +4,9 @@ from typing import Any, Optional, Union
import httpx
import litellm
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@@ -13,11 +16,9 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from ..base_aws_llm import BaseAWSLLM, Credentials
from ..common_utils import BedrockError
from ..common_utils import BedrockError, _get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
@@ -68,7 +69,7 @@ def make_sync_call(
model_response=model_response, json_mode=json_mode
)
else:
decoder = AWSEventStreamDecoder(model=model)
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
# LOGGING
@@ -279,11 +280,22 @@ class BedrockConverseLLM(BaseAWSLLM):
if _stripped.startswith(rp):
_stripped = _stripped[len(rp):]
break
# Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model")
# and capture it so it can be used as aws_region_name below.
_region_from_model: Optional[str] = None
_potential_region = _stripped.split("/", 1)[0]
if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
_region_from_model = _potential_region
_stripped = _stripped.split("/", 1)[1]
_model_for_id = _stripped
for _nova_prefix in ["nova-2/", "nova/"]:
if _stripped.startswith(_nova_prefix):
_model_for_id = _model_for_id.replace(_nova_prefix, "", 1)
break
modelId = self.encode_model_id(model_id=_model_for_id)
# Inject region extracted from model path so _get_aws_region_name picks it up
if _region_from_model is not None and "aws_region_name" not in optional_params:
optional_params["aws_region_name"] = _region_from_model
fake_stream = litellm.AmazonConverseConfig().should_fake_stream(
fake_stream=fake_stream,
litellm/llms/bedrock/chat/converse_transformation.py
+97 -35
View File
@@ -1217,15 +1217,15 @@ class AmazonConverseConfig(BaseConfig):
# Handle parallel_tool_calls configuration
parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None)
if parallel_tool_use_config is not None:
# Merge the tool_choice config from parallel_tool_calls into additional_request_params
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
for key, value in parallel_tool_use_config.items():
if key in additional_request_params and isinstance(additional_request_params[key], dict) and isinstance(value, dict):
# Merge dictionaries
additional_request_params[key].update(value)
else:
additional_request_params[key] = value
additional_request_params.pop("parallel_tool_calls", None)
# Only set the topK value in for models that support it
additional_request_params.update(
self._handle_top_k_value(model, inference_params)
@@ -1779,6 +1779,92 @@ class AmazonConverseConfig(BaseConfig):
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
@staticmethod
def _unwrap_bedrock_properties(json_str: str) -> str:
"""
Unwrap Bedrock's response_format JSON structure.
If the JSON has a single "properties" key, extract its value.
Otherwise, return the original string.
Args:
json_str: JSON string to unwrap
Returns:
Unwrapped JSON string or original if unwrapping not needed
"""
try:
response_data = json.loads(json_str)
if (
isinstance(response_data, dict)
and "properties" in response_data
and len(response_data) == 1
):
response_data = response_data["properties"]
return json.dumps(response_data)
except json.JSONDecodeError:
pass
return json_str
@staticmethod
def _filter_json_mode_tools(
json_mode: Optional[bool],
tools: List[ChatCompletionToolCallChunk],
chat_completion_message: ChatCompletionResponseMessage,
) -> Optional[List[ChatCompletionToolCallChunk]]:
"""
When json_mode is True, Bedrock may return the internal `json_tool_call`
tool alongside real user-defined tools. This method handles 3 scenarios:
1. Only json_tool_call present -> convert to text content, return None
2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools
3. No json_tool_call / no json_mode -> return tools as-is
"""
if not json_mode or not tools:
return tools if tools else None
json_tool_indices = [
i
for i, t in enumerate(tools)
if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
]
if not json_tool_indices:
# No json_tool_call found, return tools unchanged
return tools
if len(json_tool_indices) == len(tools):
# All tools are json_tool_call — convert first one to content
verbose_logger.debug(
"Processing JSON tool call response for response_format"
)
json_mode_content_str: Optional[str] = tools[0]["function"].get(
"arguments"
)
if json_mode_content_str is not None:
json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(
json_mode_content_str
)
chat_completion_message["content"] = json_mode_content_str
return None
# Mixed: filter out json_tool_call, keep real tools.
# Preserve the json_tool_call content as message text so the structured
# output from response_format is not silently lost.
first_idx = json_tool_indices[0]
json_mode_args = tools[first_idx]["function"].get("arguments")
if json_mode_args is not None:
json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(
json_mode_args
)
existing = chat_completion_message.get("content") or ""
chat_completion_message["content"] = (
existing + json_mode_args if existing else json_mode_args
)
real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
return real_tools if real_tools else None
def _transform_response( # noqa: PLR0915
self,
model: str,
@@ -1801,7 +1887,7 @@ class AmazonConverseConfig(BaseConfig):
additional_args={"complete_input_dict": data},
)
json_mode: Optional[bool] = optional_params.pop("json_mode", None)
json_mode: Optional[bool] = optional_params.get("json_mode", None)
## RESPONSE OBJECT
try:
completion_response = ConverseResponseBlock(**response.json()) # type: ignore
@@ -1885,37 +1971,13 @@ class AmazonConverseConfig(BaseConfig):
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message["content"] = content_str
if (
json_mode is True
and tools is not None
and len(tools) == 1
and tools[0]["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
):
verbose_logger.debug(
"Processing JSON tool call response for response_format"
)
json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
if json_mode_content_str is not None:
# Bedrock returns the response wrapped in a "properties" object
# We need to extract the actual content from this wrapper
try:
response_data = json.loads(json_mode_content_str)
# If Bedrock wrapped the response in "properties", extract the content
if (
isinstance(response_data, dict)
and "properties" in response_data
and len(response_data) == 1
):
response_data = response_data["properties"]
json_mode_content_str = json.dumps(response_data)
except json.JSONDecodeError:
# If parsing fails, use the original response
pass
chat_completion_message["content"] = json_mode_content_str
elif tools:
chat_completion_message["tool_calls"] = tools
filtered_tools = self._filter_json_mode_tools(
json_mode=json_mode,
tools=tools,
chat_completion_message=chat_completion_message,
)
if filtered_tools:
chat_completion_message["tool_calls"] = filtered_tools
## CALCULATING USAGE - bedrock returns usage in the headers
usage = self._transform_usage(completion_response["usage"])
litellm/llms/bedrock/chat/invoke_handler.py
+48 -16
View File
@@ -22,6 +22,7 @@ import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
@@ -252,7 +253,7 @@ async def make_call(
response.aiter_bytes(chunk_size=stream_chunk_size)
)
else:
decoder = AWSEventStreamDecoder(model=model)
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.aiter_bytes(
response.aiter_bytes(chunk_size=stream_chunk_size)
)
@@ -346,7 +347,7 @@ def make_sync_call(
response.iter_bytes(chunk_size=stream_chunk_size)
)
else:
decoder = AWSEventStreamDecoder(model=model)
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(
response.iter_bytes(chunk_size=stream_chunk_size)
)
@@ -1282,7 +1283,7 @@ def get_response_stream_shape():
class AWSEventStreamDecoder:
def __init__(self, model: str) -> None:
def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
from botocore.parsers import EventStreamJSONParser
self.model = model
@@ -1290,6 +1291,8 @@ class AWSEventStreamDecoder:
self.content_blocks: List[ContentBlockDeltaEvent] = []
self.tool_calls_index: Optional[int] = None
self.response_id: Optional[str] = None
self.json_mode = json_mode
self._current_tool_name: Optional[str] = None
def check_empty_tool_call_args(self) -> bool:
"""
@@ -1391,6 +1394,16 @@ class AWSEventStreamDecoder:
response_tool_name = get_bedrock_tool_name(
response_tool_name=_response_tool_name
)
self._current_tool_name = response_tool_name
# When json_mode is True, suppress the internal json_tool_call
# and convert its content to text in delta events instead
if (
self.json_mode is True
and response_tool_name == RESPONSE_FORMAT_TOOL_NAME
):
return tool_use, provider_specific_fields, thinking_blocks
self.tool_calls_index = (
0 if self.tool_calls_index is None else self.tool_calls_index + 1
)
@@ -1445,19 +1458,27 @@ class AWSEventStreamDecoder:
if "text" in delta_obj:
text = delta_obj["text"]
elif "toolUse" in delta_obj:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
# When json_mode is True and this is the internal json_tool_call,
# convert tool input to text content instead of tool call arguments
if (
self.json_mode is True
and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
):
text = delta_obj["toolUse"]["input"]
else:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "reasoningContent" in delta_obj:
provider_specific_fields = {
"reasoningContent": delta_obj["reasoningContent"],
@@ -1494,6 +1515,17 @@ class AWSEventStreamDecoder:
) -> Optional[ChatCompletionToolCallChunk]:
"""Handle stop/contentBlockIndex event in converse chunk parsing."""
tool_use: Optional[ChatCompletionToolCallChunk] = None
# If the ending block was the internal json_tool_call, skip emitting
# the empty-args tool chunk and reset tracking state
if (
self.json_mode is True
and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
):
self._current_tool_name = None
return tool_use
self._current_tool_name = None
is_empty = self.check_empty_tool_call_args()
if is_empty:
tool_use = {
litellm/llms/custom_httpx/llm_http_handler.py
+13
View File
@@ -1,4 +1,5 @@
import json
import ssl
from typing import (
TYPE_CHECKING,
Any,
@@ -4659,6 +4660,8 @@ class BaseLLMHTTPHandler:
api_key: Optional[str] = None,
client: Optional[Any] = None,
timeout: Optional[float] = None,
user_api_key_dict: Optional[Any] = None,
litellm_metadata: Optional[Dict[str, Any]] = None,
):
import websockets
from websockets.asyncio.client import ClientConnection
@@ -4672,6 +4675,11 @@ class BaseLLMHTTPHandler:
try:
ssl_context = get_shared_realtime_ssl_context()
if url.startswith("wss://") and ssl_context is False:
# Keep TLS for wss:// while honoring SSL_VERIFY=False semantics.
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
async with websockets.connect( # type: ignore
url,
additional_headers=headers,
@@ -4686,12 +4694,17 @@ class BaseLLMHTTPHandler:
if _session_config:
await backend_ws.send(_session_config)
_request_data: Dict[str, Any] = {}
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
realtime_streaming = RealTimeStreaming(
websocket,
cast(ClientConnection, backend_ws),
logging_obj,
provider_config,
model,
user_api_key_dict=user_api_key_dict,
request_data=_request_data,
)
if _session_config:
realtime_streaming.session_configuration_request = _session_config
litellm/llms/featherless_ai/chat/transformation.py
+6 -1
View File
@@ -103,10 +103,15 @@ class FeatherlessAIConfig(OpenAIGPTConfig):
# FeatherlessAI is openai compatible, set to custom_openai and use FeatherlessAI's endpoint
api_base = (
api_base
or get_secret_str("FEATHERLESS_AI_API_BASE")
or get_secret_str("FEATHERLESS_API_BASE")
or "https://api.featherless.ai/v1"
)
dynamic_api_key = api_key or get_secret_str("FEATHERLESS_API_KEY")
dynamic_api_key = (
api_key
or get_secret_str("FEATHERLESS_AI_API_KEY")
or get_secret_str("FEATHERLESS_API_KEY")
)
return api_base, dynamic_api_key
def validate_environment(
litellm/llms/gemini/image_generation/cost_calculator.py
+13 -1
View File
@@ -5,6 +5,9 @@ Google AI Image Generation Cost Calculator
from typing import Any
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import (
calculate_image_response_cost_from_usage,
)
from litellm.types.utils import ImageResponse
@@ -13,13 +16,22 @@ def cost_calculator(
image_response: Any,
) -> float:
"""
Vertex AI Image Generation Cost Calculator
Google AI Image Generation Cost Calculator
"""
_model_info = litellm.get_model_info(
model=model,
custom_llm_provider="gemini",
)
if isinstance(image_response, ImageResponse):
token_based_cost = calculate_image_response_cost_from_usage(
model=model,
image_response=image_response,
custom_llm_provider="gemini",
)
if token_based_cost is not None:
return token_based_cost
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if isinstance(image_response, ImageResponse):
litellm/llms/gemini/realtime/transformation.py
+49 -1
View File
@@ -829,7 +829,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
raise ValueError(f"Unknown openai event: {key}, value: {value}")
return openai_event
def transform_realtime_response(
def transform_realtime_response( # noqa: PLR0915
self,
message: Union[str, bytes],
model: str,
@@ -867,6 +867,52 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
returned_message: List[OpenAIRealtimeEvents] = []
# Handle transcription events that arrive independently from model
# content. Gemini sends inputTranscription / outputTranscription
# inside serverContent, separately from modelTurn / turnComplete.
server_content = json_message.get("serverContent")
if isinstance(server_content, dict):
input_tx = server_content.get("inputTranscription")
if isinstance(input_tx, dict) and input_tx.get("text"):
returned_message.append(
cast(OpenAIRealtimeEvents, {
"type": "conversation.item.input_audio_transcription.completed",
"event_id": "event_{}".format(uuid.uuid4()),
"transcript": input_tx["text"],
"item_id": "item_{}".format(uuid.uuid4()),
"content_index": 0,
})
)
output_tx = server_content.get("outputTranscription")
if isinstance(output_tx, dict) and output_tx.get("text"):
returned_message.append(
cast(OpenAIRealtimeEvents, {
"type": "response.audio_transcript.delta",
"event_id": "event_{}".format(uuid.uuid4()),
"delta": output_tx["text"],
"item_id": current_output_item_id or "item_{}".format(uuid.uuid4()),
"response_id": current_response_id or "resp_{}".format(uuid.uuid4()),
"output_index": 0,
"content_index": 0,
})
)
# If serverContent only contained transcription(s) and no model
# content, return early — the main loop would fail on unknown keys.
_model_content_keys = {"modelTurn", "turnComplete", "interrupted", "generationComplete"}
if not any(k in server_content for k in _model_content_keys):
return {
"response": returned_message,
"current_output_item_id": current_output_item_id,
"current_response_id": current_response_id,
"current_delta_chunks": current_delta_chunks,
"current_conversation_id": current_conversation_id,
"current_item_chunks": current_item_chunks,
"current_delta_type": current_delta_type,
"session_configuration_request": session_configuration_request,
}
for key, value in json_message.items():
# Check if this key or any nested key matches our mapping
openai_event = self.map_openai_event(
@@ -974,6 +1020,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
setup_config: BidiGenerateContentSetup = {
"model": f"models/{model}",
"generationConfig": {"responseModalities": response_modalities},
# Return input transcript so guardrails can inspect user speech.
"inputAudioTranscription": {},
}
if output_audio_transcription:
setup_config["outputAudioTranscription"] = {}
litellm/llms/hosted_vllm/responses/transformation.py
+71
View File
@@ -0,0 +1,71 @@
"""
Responses API transformation for Hosted VLLM provider.
vLLM natively supports the OpenAI-compatible /v1/responses endpoint,
so this config enables direct routing instead of falling back to
the chat completions → responses conversion pipeline.
"""
from typing import Optional
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
Configuration for Hosted VLLM Responses API support.
Extends OpenAI's config since vLLM follows OpenAI's API spec,
but uses HOSTED_VLLM_API_BASE for the base URL and defaults
to "fake-api-key" when no API key is provided (vLLM does not
require authentication by default).
"""
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.HOSTED_VLLM
def validate_environment(
self,
headers: dict,
model: str,
litellm_params: Optional[GenericLiteLLMParams],
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = (
litellm_params.api_key
or get_secret_str("HOSTED_VLLM_API_KEY")
or "fake-api-key"
) # vllm does not require an api key
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
)
return headers
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
api_base = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
if api_base is None:
raise ValueError(
"api_base not set for Hosted VLLM responses API. "
"Set via api_base parameter or HOSTED_VLLM_API_BASE environment variable"
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
# If api_base already ends with /v1, append /responses
# Otherwise append /v1/responses
if api_base.endswith("/v1"):
return f"{api_base}/responses"
return f"{api_base}/v1/responses"
litellm/llms/mistral/ocr/guardrail_translation/__init__.py
+11
View File
@@ -0,0 +1,11 @@
"""Mistral OCR handler for Unified Guardrails."""
from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler
from litellm.types.utils import CallTypes
guardrail_translation_mappings = {
CallTypes.ocr: OCRHandler,
CallTypes.aocr: OCRHandler,
}
__all__ = ["guardrail_translation_mappings", "OCRHandler"]
litellm/llms/mistral/ocr/guardrail_translation/handler.py
+155
View File
@@ -0,0 +1,155 @@
"""
OCR Handler for Unified Guardrails
Provides guardrail translation support for the OCR endpoint.
Processes the extracted markdown text from OCR pages.
"""
from typing import TYPE_CHECKING, Any, List, Optional
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.base_llm.ocr.transformation import OCRResponse
class OCRHandler(BaseTranslation):
"""
Handler for processing OCR requests/responses with guardrails.
Input: The OCR input is a document URL/reference - not text content.
We pass the document URL as text for guardrails that may want to
validate or filter document sources.
Output: OCR responses contain extracted markdown text per page.
The handler extracts all page markdown, applies guardrails,
and maps the guardrailed text back to the pages.
"""
async def process_input_messages(
self,
data: dict,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
) -> Any:
"""
Process OCR input by applying guardrails to the document reference.
The OCR input contains a document dict with a URL. We extract
the URL and pass it to the guardrail for validation.
Args:
data: Request data containing 'document' parameter
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
Returns:
Modified data with guardrails applied
"""
document = data.get("document")
if document is None or not isinstance(document, dict):
verbose_proxy_logger.debug(
"OCR guardrail: No valid document found in request data"
)
return data
# Extract the document URL for guardrail checking
texts_to_check: List[str] = []
doc_type = document.get("type")
if doc_type == "document_url":
url = document.get("document_url")
if url and isinstance(url, str):
texts_to_check.append(url)
elif doc_type == "image_url":
url = document.get("image_url")
if url and isinstance(url, str):
texts_to_check.append(url)
if not texts_to_check:
return data
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
model = data.get("model")
if model:
inputs["model"] = model
await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
return data
async def process_output_response(
self,
response: "OCRResponse",
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
user_api_key_dict: Optional[Any] = None,
) -> Any:
"""
Process OCR output by applying guardrails to extracted page text.
Extracts markdown text from each OCR page, applies guardrails,
and maps the guardrailed text back to the pages.
Args:
response: OCRResponse with pages containing markdown text
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
user_api_key_dict: User API key metadata
Returns:
Modified OCRResponse with guardrailed page text
"""
if not hasattr(response, "pages") or not response.pages:
verbose_proxy_logger.debug(
"OCR guardrail: No pages found in OCR response"
)
return response
# Extract markdown text from all pages
texts_to_check: List[str] = []
page_indices: List[int] = []
for i, page in enumerate(response.pages):
if hasattr(page, "markdown") and page.markdown:
texts_to_check.append(page.markdown)
page_indices.append(i)
if not texts_to_check:
return response
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
model = getattr(response, "model", None)
if model:
inputs["model"] = model
# Add user metadata if available
if user_api_key_dict is not None:
metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
inputs.update(metadata) # type: ignore
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data={},
input_type="response",
logging_obj=litellm_logging_obj,
)
# Map guardrailed text back to pages
guardrailed_texts = guardrailed_inputs.get("texts", [])
for idx, page_idx in enumerate(page_indices):
if idx < len(guardrailed_texts):
response.pages[page_idx].markdown = guardrailed_texts[idx]
verbose_proxy_logger.debug(
"OCR guardrail: Applied guardrail to %d pages",
len(guardrailed_texts),
)
return response
litellm/llms/openai/chat/gpt_5_transformation.py
+7 -2
View File
@@ -40,11 +40,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
gpt-5.1/5.2 support temperature when reasoning_effort="none",
unlike base gpt-5 which only supports temperature=1. Excludes
pro variants which keep stricter knobs.
pro variants which keep stricter knobs and gpt-5.2-chat variants
which only support temperature=1.
"""
model_name = model.split("/")[-1]
is_gpt_5_1 = model_name.startswith("gpt-5.1")
is_gpt_5_2 = model_name.startswith("gpt-5.2") and "pro" not in model_name
is_gpt_5_2 = (
model_name.startswith("gpt-5.2")
and "pro" not in model_name
and not model_name.startswith("gpt-5.2-chat")
)
return is_gpt_5_1 or is_gpt_5_2
@classmethod
litellm/llms/openai/openai.py
+7 -1
View File
@@ -1401,6 +1401,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
client=None,
max_retries=None,
organization: Optional[str] = None,
headers: Optional[dict] = None,
):
response = None
try:
@@ -1414,6 +1415,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
client=client,
)
if headers:
data["extra_headers"] = headers
response = await openai_aclient.images.generate(**data, timeout=timeout) # type: ignore
stringified_response = response.model_dump()
## LOGGING
@@ -1446,6 +1449,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
client=None,
aimg_generation=None,
organization: Optional[str] = None,
headers: Optional[dict] = None,
) -> ImageResponse:
data = {}
try:
@@ -1455,7 +1459,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(status_code=422, message="max retries must be an int")
if aimg_generation is True:
return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization) # type: ignore
return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization, headers=headers) # type: ignore
openai_client: OpenAI = self._get_openai_client( # type: ignore
is_async=False,
@@ -1480,6 +1484,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
## COMPLETION CALL
if headers:
data["extra_headers"] = headers
_response = openai_client.images.generate(**data, timeout=timeout) # type: ignore
response = _response.model_dump()
litellm/llms/openai_like/providers.json
+4
View File
@@ -90,5 +90,9 @@
"headers": {
"api-subscription-key": "{api_key}"
}
},
"assemblyai": {
"base_url": "https://llm-gateway.assemblyai.com/v1",
"api_key_env": "ASSEMBLYAI_API_KEY"
}
}
litellm/llms/vertex_ai/gemini/transformation.py
+2 -2
View File
@@ -500,7 +500,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
messages[msg_i]["role"] not in tool_call_message_roles
):
if len(tool_call_responses) > 0:
contents.append(ContentType(parts=tool_call_responses))
contents.append(ContentType(role="user", parts=tool_call_responses))
tool_call_responses = []
if msg_i == init_msg_i: # prevent infinite loops
@@ -510,7 +510,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
)
)
if len(tool_call_responses) > 0:
contents.append(ContentType(parts=tool_call_responses))
contents.append(ContentType(role="user", parts=tool_call_responses))
if len(contents) == 0:
verbose_logger.warning(
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+16 -1
View File
@@ -313,6 +313,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"logprobs",
"top_logprobs",
"modalities",
"audio",
"parallel_tool_calls",
"web_search_options",
]
@@ -1633,6 +1634,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
prompt_audio_tokens: Optional[int] = None
prompt_image_tokens: Optional[int] = None
prompt_text_tokens: Optional[int] = None
prompt_video_tokens: Optional[int] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
reasoning_tokens: Optional[int] = None
response_tokens: Optional[int] = None
@@ -1667,9 +1669,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
response_tokens_details.audio_tokens = token_count
elif modality == "IMAGE":
response_tokens_details.image_tokens = token_count
elif modality == "VIDEO":
response_tokens_details.video_tokens = token_count
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
# candidatesTokenCount includes all modalities, so: text = total - (image + audio)
# candidatesTokenCount includes all modalities, so: text = total - (image + audio + video)
candidates_token_count = usage_metadata.get("candidatesTokenCount", 0)
if candidates_token_count > 0:
if response_tokens_details is None:
@@ -1677,10 +1681,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if response_tokens_details.text_tokens is None:
completion_image_tokens = response_tokens_details.image_tokens or 0
completion_audio_tokens = response_tokens_details.audio_tokens or 0
completion_video_tokens = response_tokens_details.video_tokens or 0
calculated_text_tokens = (
candidates_token_count
- completion_image_tokens
- completion_audio_tokens
- completion_video_tokens
)
response_tokens_details.text_tokens = calculated_text_tokens
#########################################################
@@ -1694,12 +1700,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
prompt_text_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "IMAGE":
prompt_image_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "VIDEO":
prompt_video_tokens = detail.get("tokenCount", 0)
## Parse cacheTokensDetails (breakdown of cached tokens by modality)
## When explicit caching is used, Gemini provides this field to show which modalities were cached
cached_text_tokens: Optional[int] = None
cached_audio_tokens: Optional[int] = None
cached_image_tokens: Optional[int] = None
cached_video_tokens: Optional[int] = None
if "cacheTokensDetails" in usage_metadata:
for detail in usage_metadata["cacheTokensDetails"]:
@@ -1709,6 +1718,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
cached_text_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "IMAGE":
cached_image_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "VIDEO":
cached_video_tokens = detail.get("tokenCount", 0)
## Calculate non-cached tokens by subtracting cached from total (per modality)
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
@@ -1720,6 +1731,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
cached_tokens is not None
and prompt_text_tokens is not None
and cached_text_tokens is None
and "cacheTokensDetails" not in usage_metadata
):
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
# Subtract from text tokens since implicit caching is primarily for text content
@@ -1729,6 +1741,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and prompt_image_tokens is not None:
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
if cached_video_tokens is not None and prompt_video_tokens is not None:
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
if "thoughtsTokenCount" in usage_metadata:
reasoning_tokens = usage_metadata["thoughtsTokenCount"]
@@ -1742,6 +1756,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
audio_tokens=prompt_audio_tokens,
text_tokens=prompt_text_tokens,
image_tokens=prompt_image_tokens,
video_tokens=prompt_video_tokens,
)
completion_tokens = response_tokens or completion_response["usageMetadata"].get(
litellm/llms/vertex_ai/image_generation/cost_calculator.py
+11
View File
@@ -3,6 +3,9 @@ Vertex AI Image Generation Cost Calculator
"""
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import (
calculate_image_response_cost_from_usage,
)
from litellm.types.utils import ImageResponse
@@ -18,6 +21,14 @@ def cost_calculator(
custom_llm_provider="vertex_ai",
)
token_based_cost = calculate_image_response_cost_from_usage(
model=model,
image_response=image_response,
custom_llm_provider="vertex_ai",
)
if token_based_cost is not None:
return token_based_cost
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
litellm/llms/vertex_ai/realtime/transformation.py
+2
View File
@@ -124,6 +124,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
"silenceDurationMs": 800,
}
},
# Return input transcript so guardrails can inspect user speech.
"inputAudioTranscription": {},
# Return output transcript so clients can read what the model said.
"outputAudioTranscription": {},
}
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
+1
View File
@@ -144,6 +144,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if beta_set:
data["anthropic_beta"] = list(beta_set)
headers["anthropic-beta"] = ",".join(beta_set)
return data
litellm/llms/vertex_ai/videos/transformation.py
+36 -4
View File
@@ -119,6 +119,12 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
# Map input_reference to image (will be processed in transform_video_create_request)
if "input_reference" in video_create_optional_params:
mapped_params["image"] = video_create_optional_params["input_reference"]
elif "image" in video_create_optional_params:
mapped_params["image"] = video_create_optional_params["image"]
# Pass through a provider-specific parameters block if provided directly
if "parameters" in video_create_optional_params:
mapped_params["parameters"] = video_create_optional_params["parameters"]
# Map size to aspectRatio
if "size" in video_create_optional_params:
@@ -263,23 +269,49 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
instance_dict: Dict[str, Any] = {"prompt": prompt}
params_copy = video_create_optional_request_params.copy()
# Check if user wants to provide full instance dict
if "instances" in params_copy and isinstance(params_copy["instances"], dict):
# Replace/merge with user-provided instance
instance_dict.update(params_copy["instances"])
params_copy.pop("instances")
elif "image" in params_copy and params_copy["image"] is not None:
image_data = _convert_image_to_vertex_format(params_copy["image"])
image = params_copy["image"]
if isinstance(image, dict):
# Already in Vertex format e.g. {"gcsUri": "gs://..."} or
# {"bytesBase64Encoded": "...", "mimeType": "..."}
image_data = image
elif isinstance(image, str) and image.startswith("gs://"):
# Bare GCS URI — Vertex AI accepts gcsUri natively, no download needed
image_data = {"gcsUri": image}
elif isinstance(image, str):
raise ValueError(
f"Unsupported image value '{image}'. "
"Provide a GCS URI (gs://...), a dict with 'gcsUri' or "
"'bytesBase64Encoded'/'mimeType', or a binary file-like object."
)
else:
# File-like object — encode to base64
image_data = _convert_image_to_vertex_format(image)
instance_dict["image"] = image_data
params_copy.pop("image")
# Extract a nested "parameters" block that map_openai_params may have placed
# inside params_copy (e.g. from provider-specific pass-through). Merging it
# flat prevents the double-nesting bug:
# {"parameters": {"parameters": {...}}} ← wrong
# {"parameters": {...}} ← correct
nested_params = params_copy.pop("parameters", None)
vertex_params: Dict[str, Any] = {}
if isinstance(nested_params, dict):
vertex_params.update(nested_params)
vertex_params.update(params_copy)
# Build request data directly (TypedDict doesn't have model_dump)
request_data: Dict[str, Any] = {"instances": [instance_dict]}
# Only add parameters if there are any
if params_copy:
request_data["parameters"] = params_copy
if vertex_params:
request_data["parameters"] = vertex_params
# Append :predictLongRunning endpoint to api_base
url = f"{api_base}:predictLongRunning"
litellm/main.py
+4
View File
@@ -4680,12 +4680,16 @@ def embedding( # noqa: PLR0915
if dynamic_api_key is not None:
api_key = dynamic_api_key
allowed_openai_params: Optional[List[str]] = kwargs.get(
"allowed_openai_params", None
)
optional_params = get_optional_params_embeddings(
model=model,
user=user,
dimensions=dimensions,
encoding_format=encoding_format,
custom_llm_provider=custom_llm_provider,
allowed_openai_params=allowed_openai_params,
**non_default_params,
)
litellm/model_prices_and_context_window_backup.json
+173 -5
View File
@@ -14194,6 +14194,38 @@
"supports_vision": true,
"supports_web_search": true
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
"input_cost_per_token": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_token": 3e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
},
"deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
@@ -16257,7 +16289,7 @@
"cache_read_input_token_cost": 3e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 3e-07,
"litellm_provider": "vertex_ai-language-models",
"litellm_provider": "gemini",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"supports_reasoning": false,
@@ -19178,6 +19210,39 @@
"supports_tool_choice": true,
"supports_vision": false
},
"gpt-audio-1.5": {
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1e-05,
"supported_endpoints": [
"/v1/chat/completions"
],
"supported_modalities": [
"text",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": false,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
},
"gpt-audio-2025-08-28": {
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_token": 2.5e-06,
@@ -20895,6 +20960,38 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image": 5e-06,
"input_cost_per_token": 4e-06,
"litellm_provider": "openai",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
@@ -25060,6 +25157,25 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-opus-4.6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"openrouter/anthropic/claude-sonnet-4.5": {
"input_cost_per_image": 0.0048,
"cache_creation_input_token_cost": 3.75e-06,
@@ -26072,6 +26188,42 @@
"supports_prompt_caching": true,
"supports_computer_use": false
},
"openrouter/openrouter/auto": {
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"litellm_provider": "openrouter",
"max_input_tokens": 2000000,
"max_tokens": 2000000,
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_audio_input": true,
"supports_video_input": true
},
"openrouter/openrouter/free": {
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"litellm_provider": "openrouter",
"max_input_tokens": 200000,
"max_tokens": 200000,
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true
},
"openrouter/openrouter/bodybuilder": {
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"litellm_provider": "openrouter",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "chat"
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",
@@ -26618,8 +26770,8 @@
"mode": "chat",
"output_cost_per_token": 0.0,
"source": "https://platform.publicai.co/docs",
"supports_function_calling": true,
"supports_tool_choice": true
"supports_function_calling": false,
"supports_tool_choice": false
},
"publicai/swiss-ai/apertus-70b-instruct": {
"input_cost_per_token": 0.0,
@@ -26630,8 +26782,8 @@
"mode": "chat",
"output_cost_per_token": 0.0,
"source": "https://platform.publicai.co/docs",
"supports_function_calling": true,
"supports_tool_choice": true
"supports_function_calling": false,
"supports_tool_choice": false
},
"publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": {
"input_cost_per_token": 0.0,
@@ -31545,6 +31697,19 @@
"output_cost_per_token_batches": 6e-06,
"source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image"
},
"vertex_ai/gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
"input_cost_per_token": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_token": 3e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models"
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
@@ -32946,6 +33111,7 @@
"supports_web_search": true
},
"xai/grok-2-vision-1212": {
"deprecation_date": "2026-02-28",
"input_cost_per_image": 2e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "xai",
@@ -33050,6 +33216,7 @@
},
"xai/grok-3-mini": {
"cache_read_input_token_cost": 7.5e-08,
"deprecation_date": "2026-02-28",
"input_cost_per_token": 3e-07,
"litellm_provider": "xai",
"max_input_tokens": 131072,
@@ -33066,6 +33233,7 @@
},
"xai/grok-3-mini-beta": {
"cache_read_input_token_cost": 7.5e-08,
"deprecation_date": "2026-02-28",
"input_cost_per_token": 3e-07,
"litellm_provider": "xai",
"max_input_tokens": 131072,

Some files were not shown because too many files have changed in this diff Show More