tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-13 17:10:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into litellm_staging_01_06_2026

Browse Source
This commit is contained in:
Sameer Kankute
2026-01-09 17:06:17 +05:30
committed by GitHub
parent 0756b48963 bb9347207b
commit e7efd51bd7
298 changed files with 19816 additions and 2281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+72 -1
View File
@@ -1465,7 +1465,7 @@ jobs:
- run:
name: Run core tests
command: |
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1479,6 +1479,60 @@ jobs:
paths:
- litellm_core_tests_coverage.xml
- litellm_core_tests_coverage
litellm_mapped_tests_litellm_core_utils:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run litellm_core_utils tests
command: |
python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_core_utils_tests_coverage.xml
mv .coverage litellm_core_utils_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_core_utils_tests_coverage.xml
- litellm_core_utils_tests_coverage
litellm_mapped_tests_integrations:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run integrations tests
command: |
python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_integrations_tests_coverage.xml
mv .coverage litellm_integrations_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_integrations_tests_coverage.xml
- litellm_integrations_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
@@ -1960,6 +2014,7 @@ jobs:
- run: ruff check ./litellm
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/check_provider_folders_documented.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
@@ -3871,6 +3926,18 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_integrations:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_litellm_core_utils:
filters:
branches:
only:
- main
- /litellm_.*/
- batches_testing:
filters:
branches:
@@ -3919,6 +3986,8 @@ workflows:
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
@@ -3990,6 +4059,8 @@ workflows:
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
.github/ISSUE_TEMPLATE/bug_report.yml
+1
View File
@@ -27,6 +27,7 @@ body:
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
.github/ISSUE_TEMPLATE/feature_request.yml
+1
View File
@@ -27,6 +27,7 @@ body:
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
.github/workflows/label-component.yml
+56 -118
View File
@@ -11,134 +11,72 @@ jobs:
permissions:
issues: write
steps:
- name: Add SDK label
if: contains(github.event.issue.body, 'SDK (litellm Python package)')
- name: Add component labels
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'sdk';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
const body = context.payload.issue.body;
if (!body) return;
- name: Add Proxy label
if: contains(github.event.issue.body, 'Proxy')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'proxy';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
});
} else {
throw error;
// Define component mappings with regex patterns that handle flexible whitespace
const components = [
{
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
label: 'sdk',
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
},
{
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
label: 'proxy',
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
},
{
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
label: 'ui-dashboard',
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
},
{
pattern: /What part of LiteLLM is this about\?\s*Docs/,
label: 'docs',
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
];
- name: Add UI Dashboard label
if: contains(github.event.issue.body, 'UI Dashboard')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'ui-dashboard';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
// Find matching component
for (const component of components) {
if (component.pattern.test(body)) {
// Ensure label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label,
color: component.color,
description: component.description
});
}
}
- name: Add Docs label
if: contains(github.event.issue.body, 'Docs')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'docs';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
// Add label to issue
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
issue_number: context.issue.number,
labels: [component.label]
});
} else {
throw error;
break;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
Dockerfile
+7 -4
View File
@@ -20,7 +20,8 @@ RUN python -m pip install build
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -65,12 +66,14 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
README.md
+1 -1
View File
@@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
@@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged.
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>
ci_cd/security_scans.sh
+38 -38
View File
@@ -34,47 +34,47 @@ install_ggshield() {
echo "ggshield installed successfully"
}
# Function to run secret detection scans
run_secret_detection() {
echo "Running secret detection scans..."
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
if ! command -v ggshield &> /dev/null; then
install_ggshield
fi
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
if [ -z "$GITGUARDIAN_API_KEY" ]; then
echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
echo "ggshield requires a GitGuardian API key to scan for secrets."
echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
exit 1
fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
echo "Scanning codebase for secrets..."
echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
echo "ggshield will automatically handle rate limits and retry as needed."
echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# Use --recursive for directory scanning and auto-confirm if prompted
# .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# GITGUARDIAN_API_KEY environment variable will be used for authentication
echo y | ggshield secret scan path . --recursive || {
echo ""
echo "=========================================="
echo "ERROR: Secret Detection Failed"
echo "=========================================="
echo "ggshield has detected secrets in the codebase."
echo "Please review discovered secrets above, revoke any actively used secrets"
echo "from underlying systems and make changes to inject secrets dynamically at runtime."
echo ""
echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
echo "=========================================="
echo ""
exit 1
}
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
echo "Secret detection scans completed successfully"
}
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
@@ -209,8 +209,8 @@ main() {
install_trivy
install_grype
echo "Running secret detection scans..."
run_secret_detection
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
deploy/Dockerfile.ghcr_base
+2 -1
View File
@@ -8,7 +8,8 @@ WORKDIR /app
COPY config.yaml .
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
# Convert Windows line endings to Unix
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
docker/Dockerfile.alpine
+3 -2
View File
@@ -46,8 +46,9 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
docker/Dockerfile.custom_ui
+3 -2
View File
@@ -32,8 +32,9 @@ RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
WORKDIR /app
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
docker/Dockerfile.database
+11 -7
View File
@@ -27,7 +27,8 @@ RUN python -m pip install build
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -48,7 +49,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
WORKDIR /app
# Copy the current directory contents into the container at /app
@@ -63,20 +64,23 @@ COPY --from=builder /wheels/ /wheels/
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build Admin UI (runtime stage)
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
RUN apk add --no-cache supervisor
docker/Dockerfile.dev
+7 -2
View File
@@ -40,7 +40,8 @@ COPY enterprise/ ./enterprise/
COPY docker/ ./docker/
# Build Admin UI once
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -79,8 +80,12 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
rm -rf /wheels
# Generate prisma client and set permissions
# Convert Windows line endings to Unix for entrypoint scripts
RUN prisma generate && \
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
sed -i 's/\r$//' docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
chmod +x docker/entrypoint.sh && \
chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
docker/Dockerfile.non_root
+4 -1
View File
@@ -144,7 +144,10 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi
# Permissions, cleanup, and Prisma prep
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
docs/my-website/docs/anthropic_count_tokens.md
+1
View File
@@ -92,6 +92,7 @@ model_list:
model: vertex_ai/claude-3-5-sonnet-v2@20241022
vertex_project: my-project
vertex_location: us-east5
vertex_count_tokens_location: us-east5 # Optional: Override location for token counting (count_tokens not available on global location)
- model_name: claude-bedrock
litellm_params:
docs/my-website/docs/container_files.md
+81
View File
@@ -21,6 +21,7 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/v1/containers/{container_id}/files` | POST | Upload file to container |
| `/v1/containers/{container_id}/files` | GET | List files in container |
| `/v1/containers/{container_id}/files/{file_id}` | GET | Get file metadata |
| `/v1/containers/{container_id}/files/{file_id}/content` | GET | Download file content |
@@ -28,6 +29,45 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
## LiteLLM Python SDK
### Upload Container File
Upload files directly to a container session. This is useful when `/chat/completions` or `/responses` sends files to the container but the input file type is limited to PDF. This endpoint lets you work with other file types like CSV, Excel, Python scripts, etc.
```python showLineNumbers title="upload_container_file.py"
from litellm import upload_container_file
# Upload a CSV file
file = upload_container_file(
container_id="cntr_123...",
file=("data.csv", open("data.csv", "rb").read(), "text/csv"),
custom_llm_provider="openai"
)
print(f"Uploaded: {file.id}")
print(f"Path: {file.path}")
```
**Async:**
```python showLineNumbers title="aupload_container_file.py"
from litellm import aupload_container_file
file = await aupload_container_file(
container_id="cntr_123...",
file=("script.py", b"print('hello world')", "text/x-python"),
custom_llm_provider="openai"
)
```
**Supported file formats:**
- CSV (`.csv`)
- Excel (`.xlsx`)
- Python scripts (`.py`)
- JSON (`.json`)
- Markdown (`.md`)
- Text files (`.txt`)
- And more...
### List Container Files
```python showLineNumbers title="list_container_files.py"
@@ -103,6 +143,40 @@ print(f"Deleted: {result.deleted}")
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
### Upload File
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="upload_file.py"
from openai import OpenAI
client = OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
file = client.containers.files.create(
container_id="cntr_123...",
file=open("data.csv", "rb")
)
print(f"Uploaded: {file.id}")
print(f"Path: {file.path}")
```
</TabItem>
<TabItem value="curl" label="curl">
```bash showLineNumbers title="upload_file.sh"
curl "http://localhost:4000/v1/containers/cntr_123.../files" \
-H "Authorization: Bearer sk-1234" \
-F file="@data.csv"
```
</TabItem>
</Tabs>
### List Files
<Tabs>
@@ -236,6 +310,13 @@ curl -X DELETE "http://localhost:4000/v1/containers/cntr_123.../files/cfile_456.
## Parameters
### Upload File
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `container_id` | string | Yes | Container ID |
| `file` | FileTypes | Yes | File to upload. Can be a tuple of (filename, content, content_type), file-like object, or bytes |
### List Files
| Parameter | Type | Required | Description |
docs/my-website/docs/interactions.md
+1 -1
View File
@@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported LLM providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
| Supported LLM providers | **All LiteLLM supported CHAT COMPLETION providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
## **LiteLLM Python SDK Usage**
docs/my-website/docs/mcp_control.md
+29 -1
View File
@@ -108,7 +108,7 @@ Some MCP servers are meant to be shared broadly—think internal knowledge bases
3. Toggle **Allow All LiteLLM Keys** on.
<Image
img={require('../img/mcp_ui.png')}
img={require('../img/mcp_allow_all_ui.png')}
style={{width: '80%', display: 'block', margin: '1rem auto'}}
alt="MCP server configuration in Admin UI"
/>
@@ -634,3 +634,31 @@ Control which tools different teams can access from the same MCP server. For exa
This video shows how to set allowed tools for a Key, Team, or Organization.
<iframe width="840" height="500" src="https://www.loom.com/embed/7464d444c3324078892367272fe50745" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Dashboard View Modes
Proxy admins can also control what non-admins see inside the MCP dashboard via `general_settings.user_mcp_management_mode`:
- `restricted` *(default)* – users only see servers that their team explicitly has access to.
- `view_all` – every dashboard user can see the full MCP server list.
```yaml title="Config example"
general_settings:
user_mcp_management_mode: view_all
```
This is useful when you want discoverability for MCP offerings without granting additional execution privileges.
## Publish MCP Registry
If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry).
1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy.
2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`.
3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL.
:::note Permissions still apply
The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions.
:::
docs/my-website/docs/observability/arize_integration.md
+1
View File
@@ -68,6 +68,7 @@ environment_variables:
ARIZE_API_KEY: "141a****"
ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint
ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc)
ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name
```
2. Start the proxy
docs/my-website/docs/observability/focus.md
+93
View File
@@ -0,0 +1,93 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Focus Export (Experimental)
:::caution Experimental feature
Focus Format export is under active development and currently considered experimental.
Interfaces, schema mappings, and configuration options may change as we iterate based on user feedback.
Please treat this integration as a preview and report any issues or suggestions to help us stabilize and improve the workflow.
:::
LiteLLM can emit usage data in the [FinOps FOCUS format](https://focus.finops.org/focus-specification/v1-2/) and push artifacts (for example Parquet files) to destinations such as Amazon S3. This enables downstream cost-analysis tooling to ingest a standardised dataset directly from LiteLLM.
LiteLLM currently conforms to the FinOps FOCUS v1.2 specification when emitting this dataset.
## Overview
| Property | Details |
|----------|---------|
| Destination | Export LiteLLM usage data in FOCUS format to managed storage (currently S3) |
| Callback name | `focus` |
| Supported operations | Automatic scheduled export |
| Data format | FOCUS Normalised Dataset (Parquet) |
## Environment Variables
### Common settings
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_PROVIDER` | No | Destination provider (defaults to `s3`). |
| `FOCUS_FORMAT` | No | Output format (currently only `parquet`). |
| `FOCUS_FREQUENCY` | No | Export cadence. Prefer `hourly` or `daily` for production; `interval` is intended for short test loops. Defaults to `hourly`. |
| `FOCUS_CRON_OFFSET` | No | Minute offset used for hourly/daily cron triggers. Defaults to `5`. |
| `FOCUS_INTERVAL_SECONDS` | No | Interval (seconds) when `FOCUS_FREQUENCY="interval"`. |
| `FOCUS_PREFIX` | No | Object key prefix/folder. Defaults to `focus_exports`. |
### S3 destination
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_S3_BUCKET_NAME` | Yes | Destination bucket for exported files. |
| `FOCUS_S3_REGION_NAME` | No | AWS region for the bucket. |
| `FOCUS_S3_ENDPOINT_URL` | No | Custom endpoint (useful for S3-compatible storage). |
| `FOCUS_S3_ACCESS_KEY` | Yes | AWS access key for uploads. |
| `FOCUS_S3_SECRET_KEY` | Yes | AWS secret key for uploads. |
| `FOCUS_S3_SESSION_TOKEN` | No | AWS session token if using temporary credentials. |
## Setup via Config
### Configure environment variables
```bash
export FOCUS_PROVIDER="s3"
export FOCUS_PREFIX="focus_exports"
# S3 example
export FOCUS_S3_BUCKET_NAME="my-litellm-focus-bucket"
export FOCUS_S3_REGION_NAME="us-east-1"
export FOCUS_S3_ACCESS_KEY="AKIA..."
export FOCUS_S3_SECRET_KEY="..."
```
### Update LiteLLM config
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-your-key
litellm_settings:
callbacks: ["focus"]
```
### Start the proxy
```bash
litellm --config /path/to/config.yaml
```
During boot LiteLLM registers the Focus logger and a background job that runs according to the configured frequency.
## Planned Enhancements
- Add "Setup on UI" flow alongside the current configuration-based setup.
- Add GCS / Azure Blob to the Destination options.
- Support CSV output alongside Parquet.
## Related Links
- [Focus](https://focus.finops.org/)
docs/my-website/docs/observability/qualifire_integration.md
+122
View File
@@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
# Qualifire - LLM Evaluation, Guardrails & Observability
[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications.
**Key Features:**
- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities
- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches
- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents
- **Prompt Management** - Centralized prompt management with versioning and no-code studio
:::tip
Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more.
:::
## Pre-Requisites
1. Create an account on [Qualifire](https://app.qualifire.ai/)
2. Get your API key and webhook URL from the Qualifire dashboard
```bash
pip install litellm
```
## Quick Start
Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire.
```python
litellm.callbacks = ["qualifire_eval"]
```
```python
import litellm
import os
# Set Qualifire credentials
os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key"
os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = "your-openai-api-key"
# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire
litellm.callbacks = ["qualifire_eval"]
# OpenAI call
response = litellm.completion(
model="gpt-5",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
]
)
```
## Using with LiteLLM Proxy
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["qualifire_eval"]
general_settings:
master_key: "sk-1234"
environment_variables:
QUALIFIRE_API_KEY: "your-qualifire-api-key"
QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations"
```
2. Start the proxy
```bash
litellm --config config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
```
## Environment Variables
| Variable | Description |
| ----------------------- | ------------------------------------------------------ |
| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication |
| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard |
## What Gets Logged?
The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call.
This includes:
- Request messages and parameters
- Response content and metadata
- Token usage statistics
- Latency metrics
- Model information
- Cost data
Once data is in Qualifire, you can:
- Run evaluations to detect hallucinations, toxicity, and policy violations
- Set up guardrails to block or modify responses in real-time
- View traces across your entire AI pipeline
- Track performance and quality metrics over time
docs/my-website/docs/observability/signoz.md
+394
View File
@@ -0,0 +1,394 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# SigNoz LiteLLM Integration
For more details on setting up observability for LiteLLM, check out the [SigNoz LiteLLM observability docs](https://signoz.io/docs/litellm-observability/).
## Overview
This guide walks you through setting up observability and monitoring for LiteLLM SDK and Proxy Server using [OpenTelemetry](https://opentelemetry.io/) and exporting logs, traces, and metrics to SigNoz. With this integration, you can observe various models performance, capture request/response details, and track system-level metrics in SigNoz, giving you real-time visibility into latency, error rates, and usage trends for your LiteLLM applications.
Instrumenting LiteLLM in your AI applications with telemetry ensures full observability across your AI workflows, making it easier to debug issues, optimize performance, and understand user interactions. By leveraging SigNoz, you can analyze correlated traces, logs, and metrics in unified dashboards, configure alerts, and gain actionable insights to continuously improve reliability, responsiveness, and user experience.
## Prerequisites
- A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key
- Internet access to send telemetry data to SigNoz Cloud
- [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration
- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies
## Monitoring LiteLLM
LiteLLM can be monitored in two ways: using the **LiteLLM SDK** (directly embedded in your Python application code for programmatic LLM calls) or the **LiteLLM Proxy Server** (a standalone server that acts as a centralized gateway for managing and routing LLM requests across your infrastructure).
<Tabs>
<TabItem value="LiteLLM SDK" label="LiteLLM SDK" default>
For more detailed info on instrumenting your LiteLLM SDK applications click [here](https://docs.litellm.ai/docs/observability/opentelemetry_integration).
<Tabs>
<TabItem value="No Code" label="No Code(Recommended)" default>
No-code auto-instrumentation is recommended for quick setup with minimal code changes. It's ideal when you want to get observability up and running without modifying your application code and are leveraging standard instrumentor libraries.
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install \
opentelemetry-api \
opentelemetry-distro \
opentelemetry-exporter-otlp \
httpx \
opentelemetry-instrumentation-httpx \
litellm
```
**Step 2:** Add Automatic Instrumentation
```bash
opentelemetry-bootstrap --action=install
```
**Step 3:** Instrument your LiteLLM SDK application
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
```python
from litellm import litellm
litellm.callbacks = ["otel"]
```
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
**Step 4:** Run an example
```python
from litellm import completion, litellm
litellm.callbacks = ["otel"]
response = completion(
model="openai/gpt-4o",
messages=[{ "content": "What is SigNoz","role": "user"}]
)
print(response)
```
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
**Step 5:** Run your application with auto-instrumentation
```bash
OTEL_RESOURCE_ATTRIBUTES="service.name=<service_name>" \
OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443" \
OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>" \
OTEL_EXPORTER_OTLP_PROTOCOL=grpc \
OTEL_TRACES_EXPORTER=otlp \
OTEL_METRICS_EXPORTER=otlp \
OTEL_LOGS_EXPORTER=otlp \
OTEL_PYTHON_LOG_CORRELATION=true \
OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \
OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \
opentelemetry-instrument <your_run_command>
```
> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation.
- **`<service_name>`** is the name of your service
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
- Replace `<your_run_command>` with the actual command you would use to run your application. For example: `python main.py`
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
</TabItem>
<TabItem value="Code" label="Code" default>
Code-based instrumentation gives you fine-grained control over your telemetry configuration. Use this approach when you need to customize resource attributes, sampling strategies, or integrate with existing observability infrastructure.
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install \
opentelemetry-api \
opentelemetry-sdk \
opentelemetry-exporter-otlp \
opentelemetry-instrumentation-httpx \
opentelemetry-instrumentation-system-metrics \
litellm
```
**Step 2:** Import the necessary modules in your Python application
**Traces:**
```python
from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
```
**Logs:**
```python
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry._logs import set_logger_provider
import logging
```
**Metrics:**
```python
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry import metrics
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
```
**Step 3:** Set up the OpenTelemetry Tracer Provider to send traces directly to SigNoz Cloud
```python
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry import trace
import os
resource = Resource.create({"service.name": "<service_name>"})
provider = TracerProvider(resource=resource)
span_exporter = OTLPSpanExporter(
endpoint= os.getenv("OTEL_EXPORTER_TRACES_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
processor = BatchSpanProcessor(span_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_TRACES_ENDPOINT`** → SigNoz Cloud trace endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/traces`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 4**: Setup Logs
```python
import logging
from opentelemetry.sdk.resources import Resource
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
import os
resource = Resource.create({"service.name": "<service_name>"})
logger_provider = LoggerProvider(resource=resource)
set_logger_provider(logger_provider)
otlp_log_exporter = OTLPLogExporter(
endpoint= os.getenv("OTEL_EXPORTER_LOGS_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(otlp_log_exporter)
)
# Attach OTel logging handler to root logger
handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
logging.basicConfig(level=logging.INFO, handlers=[handler])
logger = logging.getLogger(__name__)
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_LOGS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/logs`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 5**: Setup Metrics
```python
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry import metrics
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
import os
resource = Resource.create({"service.name": "<service-name>"})
metric_exporter = OTLPMetricExporter(
endpoint= os.getenv("OTEL_EXPORTER_METRICS_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
reader = PeriodicExportingMetricReader(metric_exporter)
metric_provider = MeterProvider(metric_readers=[reader], resource=resource)
metrics.set_meter_provider(metric_provider)
meter = metrics.get_meter(__name__)
# turn on out-of-the-box metrics
SystemMetricsInstrumentor().instrument()
HTTPXClientInstrumentor().instrument()
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_METRICS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/metrics`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
> 📌 Note: SystemMetricsInstrumentor provides system metrics (CPU, memory, etc.), and HTTPXClientInstrumentor provides outbound HTTP request metrics such as request duration. If you want to add custom metrics to your LiteLLM application, see [Python Custom Metrics](https://signoz.io/opentelemetry/python-custom-metrics/).
**Step 6:** Instrument your LiteLLM application
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
```python
from litellm import litellm
litellm.callbacks = ["otel"]
```
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
**Step 7:** Run an example
```python
from litellm import completion, litellm
litellm.callbacks = ["otel"]
response = completion(
model="openai/gpt-4o",
messages=[{ "content": "What is SigNoz","role": "user"}]
)
print(response)
```
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
</TabItem>
</Tabs>
## View Traces, Logs, and Metrics in SigNoz
Your LiteLLM commands should now automatically emit traces, logs, and metrics.
You should be able to view traces in Signoz Cloud under the traces tab:
![LiteLLM SDK Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-traces.webp)
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
![LiteLLM SDK Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-traces.webp)
You should be able to view logs in Signoz Cloud under the logs tab. You can also view logs by clicking on the “Related Logs” button in the trace view to see correlated logs:
![LiteLLM SDK Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-logs.webp)
When you click on any of these logs in SigNoz, you'll see a detailed view of the log, including attributes:
![LiteLLM SDK Detailed Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-logs.webp)
You should be able to see LiteLLM related metrics in Signoz Cloud under the metrics tab:
![LiteLLM SDK Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-metrics.webp)
When you click on any of these metrics in SigNoz, you'll see a detailed view of the metric, including attributes:
![LiteLLM Detailed Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-metrics.webp)
## Dashboard
You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-sdk-dashboard/) which provides specialized visualizations for monitoring your LiteLLM usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
![LiteLLM SDK Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-sdk-dashboard.webp)
</TabItem>
<TabItem value="LiteLLM Proxy Server" label="LiteLLM Proxy Server" default>
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install opentelemetry-api \
opentelemetry-sdk \
opentelemetry-exporter-otlp \
'litellm[proxy]'
```
**Step 2:** Configure otel for the LiteLLM Proxy Server
Add the following to `config.yaml`:
```yaml
litellm_settings:
callbacks: ['otel']
```
**Step 3:** Set the following environment variables:
```bash
export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443"
export OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>"
export OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
export OTEL_TRACES_EXPORTER="otlp"
export OTEL_METRICS_EXPORTER="otlp"
export OTEL_LOGS_EXPORTER="otlp"
```
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 4:** Run the proxy server using the config file:
```bash
litellm --config config.yaml
```
Now any calls made through your LiteLLM proxy server will be traced and sent to SigNoz.
You should be able to view traces in Signoz Cloud under the traces tab:
![LiteLLM Proxy Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-traces.webp)
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
![LiteLLM Proxy Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-detailed-traces.webp)
## Dashboard
You can also check out our custom LiteLLM Proxy dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-proxy-dashboard/) which provides specialized visualizations for monitoring your LiteLLM Proxy usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
![LiteLLM Proxy Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-proxy-dashboard.webp)
</TabItem>
</Tabs>
docs/my-website/docs/providers/abliteration.md
+109
View File
@@ -0,0 +1,109 @@
# Abliteration
## Overview
| Property | Details |
|-------|-------|
| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. |
| Provider Route on LiteLLM | `abliteration/` |
| Link to Provider Doc | [Abliteration](https://abliteration.ai) |
| Base URL | `https://api.abliteration.ai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key
```
## Sample Usage
```python showLineNumbers title="Abliteration Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
)
print(response)
```
## Sample Usage - Streaming
```python showLineNumbers title="Abliteration Streaming Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Stream a short reply"}],
stream=True,
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy Server
1. Add the model to your proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: abliteration-chat
litellm_params:
model: abliteration/abliterated-model
api_key: os.environ/ABLITERATION_API_KEY
```
2. Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
## Direct API Usage (Bearer Token)
Use the environment variable as a Bearer token against the OpenAI-compatible endpoint:
`https://api.abliteration.ai/v1/chat/completions`.
```bash showLineNumbers title="cURL"
export ABLITERATION_API_KEY=""
curl https://api.abliteration.ai/v1/chat/completions \
-H "Authorization: Bearer ${ABLITERATION_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}]
}'
```
```python showLineNumbers title="Python (requests)"
import os
import requests
api_key = os.environ["ABLITERATION_API_KEY"]
response = requests.post(
"https://api.abliteration.ai/v1/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}],
},
timeout=60,
)
print(response.json())
```
docs/my-website/docs/providers/anthropic.md
+2 -2
View File
@@ -1692,9 +1692,9 @@ Assistant:
```
## Usage - PDF
## Usage - PDF
Pass base64 encoded PDF files to Anthropic models using the `image_url` field.
Pass base64 encoded PDF files to Anthropic models using the `file` content type with a `file_data` field.
<Tabs>
<TabItem value="sdk" label="SDK">
docs/my-website/docs/providers/azure_ai_img.md
+135 -2
View File
@@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Generation
# Azure AI Image Generation (Black Forest Labs - Flux)
Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
@@ -12,7 +12,7 @@ Azure AI provides powerful image generation capabilities using FLUX models from
| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/generations`](#image-generation) |
| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-editing) |
## Setup
@@ -33,6 +33,7 @@ Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
|------------|-------------|----------------|
| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
| `azure_ai/flux.2-pro` | FLUX 2 Pro model for next-generation image generation | $0.04 |
## Image Generation
@@ -85,6 +86,32 @@ print(response.data[0].url)
</TabItem>
<TabItem value="flux2" label="FLUX 2 Pro">
```python showLineNumbers title="FLUX 2 Pro Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com
# Generate image with FLUX 2 Pro
response = litellm.image_generation(
model="azure_ai/flux.2-pro",
prompt="A photograph of a red fox in an autumn forest",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
size="1024x1024",
n=1
)
print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
@@ -165,6 +192,15 @@ model_list:
model_info:
mode: image_generation
- model_name: azure-flux-2-pro
litellm_params:
model: azure_ai/flux.2-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
api_version: preview
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
@@ -239,6 +275,103 @@ curl --location 'http://localhost:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Image Editing
FLUX 2 Pro supports image editing by passing an input image along with a prompt describing the desired modifications.
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic-edit" label="Basic Image Edit">
```python showLineNumbers title="Basic Image Editing with FLUX 2 Pro"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com
# Edit an existing image
response = litellm.image_edit(
model="azure_ai/flux.2-pro",
prompt="Add a red hat to the subject",
image=open("input_image.png", "rb"),
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
)
print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images
```
</TabItem>
<TabItem value="async-edit" label="Async Image Edit">
```python showLineNumbers title="Async Image Editing"
import litellm
import asyncio
import os
async def edit_image():
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
response = await litellm.aimage_edit(
model="azure_ai/flux.2-pro",
prompt="Change the background to a sunset beach",
image=open("input_image.png", "rb"),
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
)
return response
asyncio.run(edit_image())
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
<Tabs>
<TabItem value="curl-edit" label="cURL">
```bash showLineNumbers title="Image Edit via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/edits' \
--header 'Authorization: Bearer sk-1234' \
--form 'model="azure-flux-2-pro"' \
--form 'prompt="Add sunglasses to the person"' \
--form 'image=@"input_image.png"'
```
</TabItem>
<TabItem value="openai-sdk-edit" label="OpenAI SDK">
```python showLineNumbers title="Image Edit via Proxy - OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="sk-1234"
)
response = client.images.edit(
model="azure-flux-2-pro",
prompt="Make the sky more dramatic with storm clouds",
image=open("input_image.png", "rb"),
)
print(response.data[0].b64_json)
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Generation supports the following OpenAI-compatible parameters:
docs/my-website/docs/providers/bedrock.md
+2 -1
View File
@@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Rerank Endpoint | `/rerank` |
@@ -1941,6 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Moonshot Kimi K2 Thinking | `completion(model='bedrock/moonshot.kimi-k2-thinking', messages=messages)` or `completion(model='bedrock/invoke/moonshot.kimi-k2-thinking', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding
docs/my-website/docs/providers/bedrock_imported.md
+177 -1
View File
@@ -431,4 +431,180 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"max_tokens": 300,
"temperature": 0.5
}'
```
```
### Moonshot Kimi K2 Thinking
Moonshot AI's Kimi K2 Thinking model is now available on Amazon Bedrock. This model features advanced reasoning capabilities with automatic reasoning content extraction.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/moonshot.kimi-k2-thinking`, `bedrock/invoke/moonshot.kimi-k2-thinking` |
| Provider Documentation | [AWS Bedrock Moonshot Announcement ↗](https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/) |
| Supported Parameters | `temperature`, `max_tokens`, `top_p`, `stream`, `tools`, `tool_choice` |
| Special Features | Reasoning content extraction, Tool calling |
#### Supported Features
- **Reasoning Content Extraction**: Automatically extracts `<reasoning>` tags and returns them as `reasoning_content` (similar to OpenAI's o1 models)
- **Tool Calling**: Full support for function/tool calling with tool responses
- **Streaming**: Both streaming and non-streaming responses
- **System Messages**: System message support
#### Basic Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python title="Moonshot Kimi K2 SDK Usage" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2" # or your preferred region
# Basic completion
response = completion(
model="bedrock/moonshot.kimi-k2-thinking", # or bedrock/invoke/moonshot.kimi-k2-thinking
messages=[
{"role": "user", "content": "What is 2+2? Think step by step."}
],
temperature=0.7,
max_tokens=200
)
print(response.choices[0].message.content)
# Access reasoning content if present
if response.choices[0].message.reasoning_content:
print("Reasoning:", response.choices[0].message.reasoning_content)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: kimi-k2
litellm_params:
model: bedrock/moonshot.kimi-k2-thinking
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
```
**2. Start proxy**
```bash title="Start LiteLLM Proxy" showLineNumbers
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash title="Test Kimi K2 via Proxy" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "kimi-k2",
"messages": [
{
"role": "user",
"content": "What is 2+2? Think step by step."
}
],
"temperature": 0.7,
"max_tokens": 200
}'
```
</TabItem>
</Tabs>
#### Tool Calling Example
```python title="Kimi K2 with Tool Calling" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
# Tool calling example
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "What's the weather in Tokyo?"}
],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name"
}
},
"required": ["location"]
}
}
}
]
)
if response.choices[0].message.tool_calls:
tool_call = response.choices[0].message.tool_calls[0]
print(f"Tool called: {tool_call.function.name}")
print(f"Arguments: {tool_call.function.arguments}")
```
#### Streaming Example
```python title="Kimi K2 Streaming" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "Explain quantum computing in simple terms."}
],
stream=True,
temperature=0.7
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
# Check for reasoning content in streaming
if hasattr(chunk.choices[0].delta, 'reasoning_content') and chunk.choices[0].delta.reasoning_content:
print(f"\n[Reasoning: {chunk.choices[0].delta.reasoning_content}]")
```
#### Supported Parameters
| Parameter | Type | Description | Supported |
|-----------|------|-------------|-----------|
| `temperature` | float (0-1) | Controls randomness in output | ✅ |
| `max_tokens` | integer | Maximum tokens to generate | ✅ |
| `top_p` | float | Nucleus sampling parameter | ✅ |
| `stream` | boolean | Enable streaming responses | ✅ |
| `tools` | array | Tool/function definitions | ✅ |
| `tool_choice` | string/object | Tool choice specification | ✅ |
| `stop` | array | Stop sequences | ❌ (Not supported on Bedrock) |
docs/my-website/docs/providers/gigachat.md
+283
View File
@@ -0,0 +1,283 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# GigaChat
https://developers.sber.ru/docs/ru/gigachat/api/overview
GigaChat is Sber AI's large language model, Russia's leading LLM provider.
:::tip
**We support ALL GigaChat models, just set `model=gigachat/<any-model-on-gigachat>` as a prefix when sending litellm requests**
:::
:::warning
GigaChat API uses self-signed SSL certificates. You must pass `ssl_verify=False` in your requests.
:::
## Supported Features
| Feature | Supported |
|---------|-----------|
| Chat Completion | Yes |
| Streaming | Yes |
| Async | Yes |
| Function Calling / Tools | Yes |
| Structured Output (JSON Schema) | Yes (via function call emulation) |
| Image Input | Yes (base64 and URL) - GigaChat-2-Max, GigaChat-2-Pro only |
| Embeddings | Yes |
## API Key
GigaChat uses OAuth authentication. Set your credentials as environment variables:
```python
import os
# Required: Set credentials (base64-encoded client_id:client_secret)
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
# Optional: Set scope (default is GIGACHAT_API_PERS for personal use)
os.environ['GIGACHAT_SCOPE'] = "GIGACHAT_API_PERS" # or GIGACHAT_API_B2B for business
```
Get your credentials at: https://developers.sber.ru/studio/
## Sample Usage
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[
{"role": "user", "content": "Hello from LiteLLM!"}
],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[
{"role": "user", "content": "Hello from LiteLLM!"}
],
stream=True,
ssl_verify=False, # Required for GigaChat
)
for chunk in response:
print(chunk)
```
## Sample Usage - Function Calling
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string", "description": "City name"}
},
"required": ["city"]
}
}
}]
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[{"role": "user", "content": "What's the weather in Moscow?"}],
tools=tools,
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Structured Output
GigaChat supports structured output via JSON schema (emulated through function calling):
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[{"role": "user", "content": "Extract info: John is 30 years old"}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "person",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
}
}
}
},
ssl_verify=False, # Required for GigaChat
)
print(response) # Returns JSON: {"name": "John", "age": 30}
```
## Sample Usage - Image Input
GigaChat supports image input via base64 or URL (GigaChat-2-Max and GigaChat-2-Pro only):
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max", # Vision requires GigaChat-2-Max or GigaChat-2-Pro
messages=[{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]
}],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Embeddings
```python
from litellm import embedding
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = embedding(
model="gigachat/Embeddings",
input=["Hello world", "How are you?"],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Usage with LiteLLM Proxy
### 1. Set GigaChat Models on config.yaml
```yaml
model_list:
- model_name: gigachat
litellm_params:
model: gigachat/GigaChat-2-Max
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
- model_name: gigachat-lite
litellm_params:
model: gigachat/GigaChat-2-Lite
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
- model_name: gigachat-embeddings
litellm_params:
model: gigachat/Embeddings
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
```
### 2. Start Proxy
```bash
litellm --config config.yaml
```
### 3. Test it
<Tabs>
<TabItem value="Curl" label="Curl Request">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gigachat",
"messages": [
{
"role": "user",
"content": "Hello!"
}
]
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gigachat",
messages=[{"role": "user", "content": "Hello!"}]
)
print(response)
```
</TabItem>
</Tabs>
## Supported Models
### Chat Models
| Model Name | Context Window | Vision | Description |
|------------|----------------|--------|-------------|
| gigachat/GigaChat-2-Lite | 128K | No | Fast, lightweight model |
| gigachat/GigaChat-2-Pro | 128K | Yes | Professional model with vision |
| gigachat/GigaChat-2-Max | 128K | Yes | Maximum capability model |
### Embedding Models
| Model Name | Max Input | Dimensions | Description |
|------------|-----------|------------|-------------|
| gigachat/Embeddings | 512 | 1024 | Standard embeddings |
| gigachat/Embeddings-2 | 512 | 1024 | Updated embeddings |
| gigachat/EmbeddingsGigaR | 4096 | 2560 | High-dimensional embeddings |
:::note
Available models may vary depending on your API access level (personal or business).
:::
## Limitations
- Only one function call per request (GigaChat API limitation)
- Maximum 1 image per message, 10 images total per conversation
- GigaChat API uses self-signed SSL certificates - `ssl_verify=False` is required
docs/my-website/docs/providers/llamagate.md
+228
View File
@@ -0,0 +1,228 @@
# LlamaGate
## Overview
| Property | Details |
|-------|-------|
| Description | LlamaGate is an OpenAI-compatible API gateway for open-source LLMs with credit-based billing. Access 26+ open-source models including Llama, Mistral, DeepSeek, and Qwen at competitive prices. |
| Provider Route on LiteLLM | `llamagate/` |
| Link to Provider Doc | [LlamaGate Documentation ↗](https://llamagate.dev/docs) |
| Base URL | `https://api.llamagate.dev/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage), [`/embeddings`](#embeddings) |
<br />
## What is LlamaGate?
LlamaGate provides access to open-source LLMs through an OpenAI-compatible API:
- **26+ Open-Source Models**: Llama 3.1/3.2, Mistral, Qwen, DeepSeek R1, and more
- **OpenAI-Compatible API**: Drop-in replacement for OpenAI SDK
- **Vision Models**: Qwen VL, LLaVA, olmOCR, UI-TARS for multimodal tasks
- **Reasoning Models**: DeepSeek R1, OpenThinker for complex problem-solving
- **Code Models**: CodeLlama, DeepSeek Coder, Qwen Coder, StarCoder2
- **Embedding Models**: Nomic, Qwen3 Embedding for RAG and search
- **Competitive Pricing**: $0.02-$0.55 per 1M tokens
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
```
Get your API key from [llamagate.dev](https://llamagate.dev).
## Supported Models
### General Purpose
| Model | Model ID |
|-------|----------|
| Llama 3.1 8B | `llamagate/llama-3.1-8b` |
| Llama 3.2 3B | `llamagate/llama-3.2-3b` |
| Mistral 7B v0.3 | `llamagate/mistral-7b-v0.3` |
| Qwen 3 8B | `llamagate/qwen3-8b` |
| Dolphin 3 8B | `llamagate/dolphin3-8b` |
### Reasoning Models
| Model | Model ID |
|-------|----------|
| DeepSeek R1 8B | `llamagate/deepseek-r1-8b` |
| DeepSeek R1 Distill Qwen 7B | `llamagate/deepseek-r1-7b-qwen` |
| OpenThinker 7B | `llamagate/openthinker-7b` |
### Code Models
| Model | Model ID |
|-------|----------|
| Qwen 2.5 Coder 7B | `llamagate/qwen2.5-coder-7b` |
| DeepSeek Coder 6.7B | `llamagate/deepseek-coder-6.7b` |
| CodeLlama 7B | `llamagate/codellama-7b` |
| CodeGemma 7B | `llamagate/codegemma-7b` |
| StarCoder2 7B | `llamagate/starcoder2-7b` |
### Vision Models
| Model | Model ID |
|-------|----------|
| Qwen 3 VL 8B | `llamagate/qwen3-vl-8b` |
| LLaVA 1.5 7B | `llamagate/llava-7b` |
| Gemma 3 4B | `llamagate/gemma3-4b` |
| olmOCR 7B | `llamagate/olmocr-7b` |
| UI-TARS 1.5 7B | `llamagate/ui-tars-7b` |
### Embedding Models
| Model | Model ID |
|-------|----------|
| Nomic Embed Text | `llamagate/nomic-embed-text` |
| Qwen 3 Embedding 8B | `llamagate/qwen3-embedding-8b` |
| EmbeddingGemma 300M | `llamagate/embeddinggemma-300m` |
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="LlamaGate Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# LlamaGate call
response = completion(
model="llamagate/llama-3.1-8b",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="LlamaGate Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# LlamaGate call with streaming
response = completion(
model="llamagate/llama-3.1-8b",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Vision
```python showLineNumbers title="LlamaGate Vision Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]
}
]
# LlamaGate vision call
response = completion(
model="llamagate/qwen3-vl-8b",
messages=messages
)
print(response)
```
### Embeddings
```python showLineNumbers title="LlamaGate Embeddings"
import os
import litellm
from litellm import embedding
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
# LlamaGate embedding call
response = embedding(
model="llamagate/nomic-embed-text",
input=["Hello world", "How are you?"]
)
print(response)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export LLAMAGATE_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: llama-3.1-8b
litellm_params:
model: llamagate/llama-3.1-8b
api_key: os.environ/LLAMAGATE_API_KEY
- model_name: deepseek-r1
litellm_params:
model: llamagate/deepseek-r1-8b
api_key: os.environ/LLAMAGATE_API_KEY
- model_name: qwen-coder
litellm_params:
model: llamagate/qwen2.5-coder-7b
api_key: os.environ/LLAMAGATE_API_KEY
```
## Supported OpenAI Parameters
LlamaGate supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature (0-2) |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. JSON mode or JSON schema |
## Pricing
LlamaGate offers competitive per-token pricing:
| Model Category | Input (per 1M) | Output (per 1M) |
|----------------|----------------|-----------------|
| Embeddings | $0.02 | - |
| Small (3-4B) | $0.03-$0.04 | $0.08 |
| Medium (7-8B) | $0.03-$0.15 | $0.05-$0.55 |
| Code Models | $0.06-$0.10 | $0.12-$0.20 |
| Reasoning | $0.08-$0.10 | $0.15-$0.20 |
## Additional Resources
- [LlamaGate Documentation](https://llamagate.dev/docs)
- [LlamaGate Pricing](https://llamagate.dev/pricing)
- [LlamaGate API Reference](https://llamagate.dev/docs/api)
docs/my-website/docs/providers/manus.md
+194
View File
@@ -0,0 +1,194 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Manus
Use Manus AI agents through LiteLLM's OpenAI-compatible Responses API.
| Property | Details |
|----------|---------|
| Description | Manus is an AI agent platform for complex reasoning tasks, document analysis, and multi-step workflows with asynchronous task execution. |
| Provider Route on LiteLLM | `manus/{agent_profile}` |
| Supported Operations | `/responses` (Responses API) |
| Provider Doc | [Manus API ↗](https://open.manus.im/docs/openai-compatibility) |
## Model Format
```shell
manus/{agent_profile}
```
**Examples:**
- `manus/manus-1.6` - General purpose agent
- `manus/manus-1.6-lite` - Lightweight agent for simple tasks
- `manus/manus-1.6-max` - Advanced agent for complex analysis
## LiteLLM Python SDK
```python showLineNumbers title="Basic Usage"
import litellm
import os
import time
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Create task
response = litellm.responses(
model="manus/manus-1.6",
input="What's the capital of France?",
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = litellm.get_response(
response_id=task_id,
custom_llm_provider="manus",
)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
## LiteLLM AI Gateway
### Setup
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: manus-agent
litellm_params:
model: manus/manus-1.6
api_key: os.environ/MANUS_API_KEY
```
```bash title="Start Proxy"
litellm --config config.yaml
```
### Usage
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Create Task"
# Create task
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": "What is the capital of France?"
}'
# Response
{
"id": "task_abc123",
"status": "running",
"metadata": {
"task_url": "https://manus.im/app/task_abc123"
}
}
```
```bash showLineNumbers title="Poll for Completion"
# Check status (repeat until status is "completed")
curl http://localhost:4000/responses/task_abc123 \
-H "Authorization: Bearer your-proxy-key"
# When completed
{
"id": "task_abc123",
"status": "completed",
"output": [
{
"role": "user",
"content": [{"text": "What is the capital of France?"}]
},
{
"role": "assistant",
"content": [{"text": "The capital of France is Paris."}]
}
]
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Create Task and Poll"
import openai
import time
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Create task
response = client.responses.create(
model="manus-agent",
input="What is the capital of France?"
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = client.responses.retrieve(response_id=task_id)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
</TabItem>
</Tabs>
## How It Works
Manus operates as an **asynchronous agent API**:
1. **Create Task**: When you call `litellm.responses()`, Manus creates a task and returns immediately with `status: "running"`
2. **Task Executes**: The agent works on your request in the background
3. **Poll for Completion**: You must repeatedly call `litellm.get_response()` or `client.responses.retrieve()` until the status changes to `"completed"`
4. **Get Results**: Once completed, the `output` field contains the full conversation
**Task Statuses:**
- `running` - Agent is actively working
- `pending` - Agent is waiting for input
- `completed` - Task finished successfully
- `error` - Task failed
:::tip Production Usage
For production applications, use [webhooks](https://open.manus.im/docs/webhooks) instead of polling to get notified when tasks complete.
:::
## Supported Parameters
| Parameter | Supported | Notes |
|-----------|-----------|-------|
| `input` | ✅ | Text, images, or structured content |
| `stream` | ✅ | Fake streaming (task runs async) |
| `max_output_tokens` | ✅ | Limits response length |
| `previous_response_id` | ✅ | For multi-turn conversations |
## Related Documentation
- [LiteLLM Responses API](/docs/response_api)
- [Manus OpenAI Compatibility](https://open.manus.im/docs/openai-compatibility)
docs/my-website/docs/providers/vertex.md
+41 -4
View File
@@ -35,6 +35,8 @@ import json
# !gcloud auth application-default login - run this to add vertex credentials to your env
## OR ##
file_path = 'path/to/vertex_ai_service_account.json'
## OR ##
export VERTEXAI_API_KEY="your-api-key"
# Load the JSON file
with open(file_path, 'r') as file:
@@ -47,7 +49,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai/gemini-2.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
vertex_credentials=vertex_credentials_json # Can remove this is added VERTEXAI_API_KEY in env
)
```
@@ -1329,15 +1331,41 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
## Authentication - vertex_project, vertex_location, etc.
LiteLLM supports two authentication methods for Vertex AI:
1. **API Key Authentication** (Recommended for getting started)
2. **Service Account Credentials** (Recommended for production)
Set your vertex credentials via:
- dynamic params
OR
- env vars
### **Authentication Method 1:
### **Dynamic Params**
The simplest way to authenticate with Vertex AI. You can set:
- `api_key` (str) - Your Vertex AI API key
You can set:
**Environment Variables:**
```bash
export VERTEXAI_API_KEY="your-api-key"
```
**Or pass as parameters:**
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-2.0-flash-exp",
messages=[{"role": "user", "content": "Hello!"}],
api_key="your-vertex-api-key",
)
```
### **Authentication Method 2: Service Account Credentials**
For production environments with fine-grained access control. You can set:
- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json
- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.). Some models support the global location, please see [Vertex AI documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#supported_models)
- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials
@@ -1392,7 +1420,16 @@ model_list:
### **Environment Variables**
You can set:
#### For API Key Authentication:
- `VERTEXAI_API_KEY` or `VERTEX_API_KEY` - Your Vertex AI API key
```bash
export VERTEXAI_API_KEY="your-vertex-api-key"
```
#### For Service Account Authentication:
- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly).
- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials
docs/my-website/docs/proxy/caching.md
+200 -115
View File
@@ -1,28 +1,29 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem';
# Caching
# Caching
:::note
:::note
For OpenAI/Anthropic Prompt Caching, go [here](../completion/prompt_caching.md)
:::
Cache LLM Responses. LiteLLM's caching system stores and reuses LLM responses to save costs and reduce latency. When you make the same request twice, the cached response is returned instead of calling the LLM API again.
Cache LLM Responses. LiteLLM's caching system stores and reuses LLM responses to save costs and
reduce latency. When you make the same request twice, the cached response is returned instead of
calling the LLM API again.
### Supported Caches
- In Memory Cache
- Disk Cache
- Redis Cache
- Redis Cache
- Qdrant Semantic Cache
- Redis Semantic Cache
- s3 Bucket Cache
- S3 Bucket Cache
- GCS Bucket Cache
## Quick Start
<Tabs>
<TabItem value="redis" label="redis cache">
@@ -30,6 +31,7 @@ Cache LLM Responses. LiteLLM's caching system stores and reuses LLM responses to
Caching can be enabled by adding the `cache` key in the `config.yaml`
#### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
@@ -41,18 +43,19 @@ model_list:
litellm_settings:
set_verbose: True
cache: True # set cache responses to True, litellm defaults to using a redis cache
cache: True # set cache responses to True, litellm defaults to using a redis cache
```
#### [OPTIONAL] Step 1.5: Add redis namespaces, default ttl
#### [OPTIONAL] Step 1.5: Add redis namespaces, default ttl
#### Namespace
If you want to create some folder for your keys, you can set a namespace, like this:
```yaml
litellm_settings:
cache: true
cache_params: # set cache params for redis
cache: true
cache_params: # set cache params for redis
type: redis
namespace: "litellm.caching.caching"
```
@@ -63,7 +66,7 @@ and keys will be stored like:
litellm.caching.caching:<hash>
```
#### Redis Cluster
#### Redis Cluster
<Tabs>
@@ -75,12 +78,11 @@ model_list:
litellm_params:
model: "*"
litellm_settings:
cache: True
cache_params:
type: redis
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
redis_startup_nodes: [{ "host": "127.0.0.1", "port": "7001" }]
```
</TabItem>
@@ -121,8 +123,7 @@ print("REDIS_CLUSTER_NODES", os.environ["REDIS_CLUSTER_NODES"])
</Tabs>
#### Redis Sentinel
#### Redis Sentinel
<Tabs>
@@ -134,7 +135,6 @@ model_list:
litellm_params:
model: "*"
litellm_settings:
cache: true
cache_params:
@@ -181,18 +181,17 @@ print("REDIS_SENTINEL_NODES", os.environ["REDIS_SENTINEL_NODES"])
```yaml
litellm_settings:
cache: true
cache_params: # set cache params for redis
cache: true
cache_params: # set cache params for redis
type: redis
ttl: 600 # will be cached on redis for 600s
# default_in_memory_ttl: Optional[float], default is None. time in seconds.
# default_in_redis_ttl: Optional[float], default is None. time in seconds.
# default_in_memory_ttl: Optional[float], default is None. time in seconds.
# default_in_redis_ttl: Optional[float], default is None. time in seconds.
```
#### SSL
just set `REDIS_SSL="True"` in your .env, and LiteLLM will pick this up.
just set `REDIS_SSL="True"` in your .env, and LiteLLM will pick this up.
```env
REDIS_SSL="True"
@@ -204,14 +203,14 @@ For quick testing, you can also use REDIS_URL, eg.:
REDIS_URL="rediss://.."
```
but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between using it vs. redis_host, port, etc.
but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between
using it vs. redis_host, port, etc.
#### GCP IAM Authentication
For GCP Memorystore Redis with IAM authentication, install the required dependency:
:::info
IAM authentication for redis is only supported via GCP and only on Redis Clusters for now.
:::info IAM authentication for redis is only supported via GCP and only on Redis Clusters for now.
:::
```shell
@@ -229,7 +228,8 @@ litellm_settings:
cache: True
cache_params:
type: redis
redis_startup_nodes: [{"host": "10.128.0.2", "port": 6379}, {"host": "10.128.0.2", "port": 11008}]
redis_startup_nodes:
[{ "host": "10.128.0.2", "port": 6379 }, { "host": "10.128.0.2", "port": 11008 }]
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
ssl: true
ssl_cert_reqs: null
@@ -242,7 +242,6 @@ litellm_settings:
You can configure GCP IAM Redis authentication in your .env:
For Redis Cluster:
```env
@@ -283,24 +282,29 @@ Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable cac
```
**Additional kwargs**
You can pass in any additional redis.Redis arg, by storing the variable + value in your os environment, like this:
You can pass in any additional redis.Redis arg, by storing the variable + value in your os
environment, like this:
```shell
REDIS_<redis-kwarg-name> = ""
```
```
[**See how it's read from the environment**](https://github.com/BerriAI/litellm/blob/4d7ff1b33b9991dcf38d821266290631d9bcd2dd/litellm/_redis.py#L40)
#### Step 3: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
</TabItem>
</TabItem>
<TabItem value="qdrant-semantic" label="Qdrant Semantic cache">
Caching can be enabled by adding the `cache` key in the `config.yaml`
#### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: fake-openai-endpoint
@@ -315,13 +319,13 @@ model_list:
litellm_settings:
set_verbose: True
cache: True # set cache responses to True, litellm defaults to using a redis cache
cache: True # set cache responses to True, litellm defaults to using a redis cache
cache_params:
type: qdrant-semantic
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
qdrant_quantization_config: binary
similarity_threshold: 0.8 # similarity threshold for semantic cache
similarity_threshold: 0.8 # similarity threshold for semantic cache
```
#### Step 2: Add Qdrant Credentials to your .env
@@ -332,11 +336,11 @@ QDRANT_API_BASE = "https://5392d382-45*********.cloud.qdrant.io"
```
#### Step 3: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
#### Step 4. Test it
```shell
@@ -351,13 +355,15 @@ curl -i http://localhost:4000/v1/chat/completions \
}'
```
**Expect to see `x-litellm-semantic-similarity` in the response headers when semantic caching is one**
**Expect to see `x-litellm-semantic-similarity` in the response headers when semantic caching is
one**
</TabItem>
<TabItem value="s3" label="s3 cache">
#### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
@@ -369,28 +375,70 @@ model_list:
litellm_settings:
set_verbose: True
cache: True # set cache responses to True
cache_params: # set cache params for s3
cache: True # set cache responses to True
cache_params: # set cache params for s3
type: s3
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
```
#### Step 2: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
</TabItem>
<TabItem value="gcs" label="gcs cache">
#### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
- model_name: text-embedding-ada-002
litellm_params:
model: text-embedding-ada-002
litellm_settings:
set_verbose: True
cache: True # set cache responses to True
cache_params: # set cache params for gcs
type: gcs
gcs_bucket_name: cache-bucket-litellm # GCS Bucket Name for caching
gcs_path_service_account: os.environ/GCS_PATH_SERVICE_ACCOUNT # use os.environ/<variable name> to pass environment variables. This is the path to your GCS service account JSON file
gcs_path: cache/ # [OPTIONAL] GCS path prefix for cache objects
```
#### Step 2: Add GCS Credentials to .env
Set the GCS environment variables in your .env file:
```shell
GCS_BUCKET_NAME="your-gcs-bucket-name"
GCS_PATH_SERVICE_ACCOUNT="/path/to/service-account.json"
```
#### Step 3: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
</TabItem>
<TabItem value="redis-sem" label="redis semantic cache">
Caching can be enabled by adding the `cache` key in the `config.yaml`
#### Step 1: Add `cache` to the config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
@@ -405,40 +453,45 @@ model_list:
litellm_settings:
set_verbose: True
cache: True # set cache responses to True
cache: True # set cache responses to True
cache_params:
type: "redis-semantic"
similarity_threshold: 0.8 # similarity threshold for semantic cache
type: "redis-semantic"
similarity_threshold: 0.8 # similarity threshold for semantic cache
redis_semantic_cache_embedding_model: azure-embedding-model # set this to a model_name set in model_list
```
#### Step 2: Add Redis Credentials to .env
Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable caching.
```shell
REDIS_URL = "" # REDIS_URL='redis://username:password@hostname:port/database'
## OR ##
REDIS_HOST = "" # REDIS_HOST='redis-18841.c274.us-east-1-3.ec2.cloud.redislabs.com'
REDIS_PORT = "" # REDIS_PORT='18841'
REDIS_PASSWORD = "" # REDIS_PASSWORD='liteLlmIsAmazing'
```
```shell
REDIS_URL = "" # REDIS_URL='redis://username:password@hostname:port/database'
## OR ##
REDIS_HOST = "" # REDIS_HOST='redis-18841.c274.us-east-1-3.ec2.cloud.redislabs.com'
REDIS_PORT = "" # REDIS_PORT='18841'
REDIS_PASSWORD = "" # REDIS_PASSWORD='liteLlmIsAmazing'
```
**Additional kwargs**
You can pass in any additional redis.Redis arg, by storing the variable + value in your os environment, like this:
You can pass in any additional redis.Redis arg, by storing the variable + value in your os
environment, like this:
```shell
REDIS_<redis-kwarg-name> = ""
```
```
#### Step 3: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
</TabItem>
</TabItem>
<TabItem value="local" label="In Memory Cache">
#### Step 1: Add `cache` to the config.yaml
```yaml
litellm_settings:
cache: True
@@ -447,6 +500,7 @@ litellm_settings:
```
#### Step 2: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
@@ -456,15 +510,17 @@ $ litellm --config /path/to/config.yaml
<TabItem value="disk" label="Disk Cache">
#### Step 1: Add `cache` to the config.yaml
```yaml
litellm_settings:
cache: True
cache_params:
type: disk
disk_cache_dir: /tmp/litellm-cache # OPTIONAL, default to ./.litellm_cache
disk_cache_dir: /tmp/litellm-cache # OPTIONAL, default to ./.litellm_cache
```
#### Step 2: Run proxy with config
```shell
$ litellm --config /path/to/config.yaml
```
@@ -473,7 +529,6 @@ $ litellm --config /path/to/config.yaml
</Tabs>
## Usage
### Basic
@@ -482,6 +537,7 @@ $ litellm --config /path/to/config.yaml
<TabItem value="chat_completions" label="/chat/completions">
Send the same request twice:
```shell
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
@@ -499,10 +555,12 @@ curl http://0.0.0.0:4000/v1/chat/completions \
"temperature": 0.7
}'
```
</TabItem>
<TabItem value="embeddings" label="/embeddings">
Send the same request twice:
```shell
curl --location 'http://0.0.0.0:4000/embeddings' \
--header 'Content-Type: application/json' \
@@ -518,18 +576,19 @@ curl --location 'http://0.0.0.0:4000/embeddings' \
"input": ["write a litellm poem"]
}'
```
</TabItem>
</Tabs>
### Dynamic Cache Controls
| Parameter | Type | Description |
|-----------|------|-------------|
| `ttl` | *Optional(int)* | Will cache the response for the user-defined amount of time (in seconds) |
| `s-maxage` | *Optional(int)* | Will only accept cached responses that are within user-defined range (in seconds) |
| `no-cache` | *Optional(bool)* | Will not store the response in cache. |
| `no-store` | *Optional(bool)* | Will not cache the response |
| `namespace` | *Optional(str)* | Will cache the response under a user-defined namespace |
| Parameter | Type | Description |
| ----------- | ---------------- | --------------------------------------------------------------------------------- |
| `ttl` | _Optional(int)_ | Will cache the response for the user-defined amount of time (in seconds) |
| `s-maxage` | _Optional(int)_ | Will only accept cached responses that are within user-defined range (in seconds) |
| `no-cache` | _Optional(bool)_ | Will not store the response in cache. |
| `no-store` | _Optional(bool)_ | Will not cache the response |
| `namespace` | _Optional(str)_ | Will cache the response under a user-defined namespace |
Each cache parameter can be controlled on a per-request basis. Here are examples for each parameter:
@@ -558,6 +617,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -574,6 +634,7 @@ curl http://localhost:4000/v1/chat/completions \
]
}'
```
</TabItem>
</Tabs>
@@ -602,6 +663,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -618,10 +680,12 @@ curl http://localhost:4000/v1/chat/completions \
]
}'
```
</TabItem>
</Tabs>
### `no-cache`
Force a fresh response, bypassing the cache.
<Tabs>
@@ -645,6 +709,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -661,6 +726,7 @@ curl http://localhost:4000/v1/chat/completions \
]
}'
```
</TabItem>
</Tabs>
@@ -668,7 +734,6 @@ curl http://localhost:4000/v1/chat/completions \
Will not store the response in cache.
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
@@ -690,6 +755,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -706,10 +772,12 @@ curl http://localhost:4000/v1/chat/completions \
]
}'
```
</TabItem>
</Tabs>
### `namespace`
Store the response under a specific cache namespace.
<Tabs>
@@ -733,6 +801,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -749,36 +818,37 @@ curl http://localhost:4000/v1/chat/completions \
]
}'
```
</TabItem>
</Tabs>
## Set cache for proxy, but not on the actual llm api call
Use this if you just want to enable features like rate limiting, and loadbalancing across multiple instances.
Set `supported_call_types: []` to disable caching on the actual api call.
Use this if you just want to enable features like rate limiting, and loadbalancing across multiple
instances.
Set `supported_call_types: []` to disable caching on the actual api call.
```yaml
litellm_settings:
cache: True
cache_params:
type: redis
supported_call_types: []
supported_call_types: []
```
## Debugging Caching - `/cache/ping`
LiteLLM Proxy exposes a `/cache/ping` endpoint to test if the cache is working as expected
**Usage**
```shell
curl --location 'http://0.0.0.0:4000/cache/ping' -H "Authorization: Bearer sk-1234"
```
**Expected Response - when cache healthy**
```shell
{
"status": "healthy",
@@ -803,7 +873,8 @@ curl --location 'http://0.0.0.0:4000/cache/ping' -H "Authorization: Bearer sk-1
### Control Call Types Caching is on for - (`/chat/completion`, `/embeddings`, etc.)
By default, caching is on for all call types. You can control which call types caching is on for by setting `supported_call_types` in `cache_params`
By default, caching is on for all call types. You can control which call types caching is on for by
setting `supported_call_types` in `cache_params`
**Cache will only be on for the call types specified in `supported_call_types`**
@@ -812,10 +883,13 @@ litellm_settings:
cache: True
cache_params:
type: redis
supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
supported_call_types:
["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
```
### Set Cache Params on config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
@@ -827,22 +901,25 @@ model_list:
litellm_settings:
set_verbose: True
cache: True # set cache responses to True, litellm defaults to using a redis cache
cache_params: # cache_params are optional
type: "redis" # The type of cache to initialize. Can be "local" or "redis". Defaults to "local".
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
cache: True # set cache responses to True, litellm defaults to using a redis cache
cache_params: # cache_params are optional
type: "redis" # The type of cache to initialize. Can be "local", "redis", "s3", or "gcs". Defaults to "local".
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
# Optional configurations
supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
supported_call_types:
["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
```
### Deleting Cache Keys - `/cache/delete`
### Deleting Cache Keys - `/cache/delete`
In order to delete a cache key, send a request to `/cache/delete` with the `keys` you want to delete
Example
Example
```shell
curl -X POST "http://0.0.0.0:4000/cache/delete" \
-H "Authorization: Bearer sk-1234" \
@@ -854,7 +931,10 @@ curl -X POST "http://0.0.0.0:4000/cache/delete" \
```
#### Viewing Cache Keys from responses
You can view the cache_key in the response headers, on cache hits the cache key is sent as the `x-litellm-cache-key` response headers
You can view the cache_key in the response headers, on cache hits the cache key is sent as the
`x-litellm-cache-key` response headers
```shell
curl -i --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
@@ -871,7 +951,8 @@ curl -i --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
Response from litellm proxy
Response from litellm proxy
```json
date: Thu, 04 Apr 2024 17:37:21 GMT
content-type: application/json
@@ -891,7 +972,7 @@ x-litellm-cache-key: 586bf3f3c1bf5aecb55bd9996494d3bbc69eb58397163add6d49537762a
],
"created": 1712252235,
}
```
### **Set Caching Default Off - Opt in only **
@@ -916,7 +997,6 @@ litellm_settings:
2. **Opting in to cache when cache is default off**
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
@@ -939,6 +1019,7 @@ chat_completion = client.chat.completions.create(
}
)
```
</TabItem>
<TabItem value="curl" label="curl">
@@ -977,45 +1058,49 @@ litellm_settings:
```yaml
cache_params:
# ttl
# ttl
ttl: Optional[float]
default_in_memory_ttl: Optional[float]
default_in_redis_ttl: Optional[float]
max_connections: Optional[Int]
# Type of cache (options: "local", "redis", "s3")
# Type of cache (options: "local", "redis", "s3", "gcs")
type: s3
# List of litellm call types to cache for
# Options: "completion", "acompletion", "embedding", "aembedding"
supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
supported_call_types:
["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
# Redis cache parameters
host: localhost # Redis server hostname or IP address
port: "6379" # Redis server port (as a string)
password: secret_password # Redis server password
host: localhost # Redis server hostname or IP address
port: "6379" # Redis server port (as a string)
password: secret_password # Redis server password
namespace: Optional[str] = None,
# GCP IAM Authentication for Redis
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
# S3 cache parameters
s3_bucket_name: your_s3_bucket_name # Name of the S3 bucket
s3_region_name: us-west-2 # AWS region of the S3 bucket
s3_api_version: 2006-03-01 # AWS S3 API version
s3_use_ssl: true # Use SSL for S3 connections (options: true, false)
s3_verify: true # SSL certificate verification for S3 connections (options: true, false)
s3_endpoint_url: https://s3.amazonaws.com # S3 endpoint URL
s3_aws_access_key_id: your_access_key # AWS Access Key ID for S3
s3_aws_secret_access_key: your_secret_key # AWS Secret Access Key for S3
s3_aws_session_token: your_session_token # AWS Session Token for temporary credentials
s3_bucket_name: your_s3_bucket_name # Name of the S3 bucket
s3_region_name: us-west-2 # AWS region of the S3 bucket
s3_api_version: 2006-03-01 # AWS S3 API version
s3_use_ssl: true # Use SSL for S3 connections (options: true, false)
s3_verify: true # SSL certificate verification for S3 connections (options: true, false)
s3_endpoint_url: https://s3.amazonaws.com # S3 endpoint URL
s3_aws_access_key_id: your_access_key # AWS Access Key ID for S3
s3_aws_secret_access_key: your_secret_key # AWS Secret Access Key for S3
s3_aws_session_token: your_session_token # AWS Session Token for temporary credentials
# GCS cache parameters
gcs_bucket_name: your_gcs_bucket_name # Name of the GCS bucket
gcs_path_service_account: /path/to/service-account.json # Path to GCS service account JSON file
gcs_path: cache/ # [OPTIONAL] GCS path prefix for cache objects
```
## Provider-Specific Optional Parameters Caching
docs/my-website/docs/proxy/config_settings.md
+73 -43
View File
@@ -24,9 +24,8 @@ litellm_settings:
turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data.
redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
# Networking settings
request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
force_ipv4: boolean # If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API
# Debugging - see debugging docs for more options
@@ -35,63 +34,71 @@ litellm_settings:
# Fallbacks, reliability
default_fallbacks: ["claude-opus"] # set default_fallbacks, in case a specific model group is misconfigured / bad.
content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors
context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors
content_policy_fallbacks: [{ "gpt-3.5-turbo-small": ["claude-opus"] }] # fallbacks for ContentPolicyErrors
context_window_fallbacks: [{ "gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"] }] # fallbacks for ContextWindowExceededErrors
# MCP Aliases - Map aliases to MCP server names for easier tool access
mcp_aliases: { "github": "github_mcp_server", "zapier": "zapier_mcp_server", "deepwiki": "deepwiki_mcp_server" } # Maps friendly aliases to MCP server names. Only the first alias for each server is used
mcp_aliases: {
"github": "github_mcp_server",
"zapier": "zapier_mcp_server",
"deepwiki": "deepwiki_mcp_server",
} # Maps friendly aliases to MCP server names. Only the first alias for each server is used
# Caching settings
cache: true
cache_params: # set cache params for redis
type: redis # type of cache to initialize
cache: true
cache_params: # set cache params for redis
type: redis # type of cache to initialize (options: "local", "redis", "s3", "gcs")
# Optional - Redis Settings
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
namespace: "litellm.caching.caching" # namespace for redis cache
max_connections: 100 # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py.
# Optional - Redis Cluster Settings
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
redis_startup_nodes: [{ "host": "127.0.0.1", "port": "7001" }]
# Optional - Redis Sentinel Settings
service_name: "mymaster"
sentinel_nodes: [["localhost", 26379]]
# Optional - GCP IAM Authentication for Redis
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
# Optional - Qdrant Semantic Cache Settings
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
qdrant_quantization_config: binary
similarity_threshold: 0.8 # similarity threshold for semantic cache
similarity_threshold: 0.8 # similarity threshold for semantic cache
# Optional - S3 Cache Settings
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
# Optional - GCS Cache Settings
gcs_bucket_name: cache-bucket-litellm # GCS Bucket Name for caching
gcs_path_service_account: os.environ/GCS_PATH_SERVICE_ACCOUNT # Path to GCS service account JSON file
gcs_path: cache/ # [OPTIONAL] GCS path prefix for cache objects
# Common Cache settings
# Optional - Supported call types for caching
supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
supported_call_types:
["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
callback_settings:
otel:
message_logging: boolean # OTEL logging callback specific settings
message_logging: boolean # OTEL logging callback specific settings
general_settings:
completion_model: string
@@ -111,6 +118,7 @@ general_settings:
master_key: string
maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion.
maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in.
user_mcp_management_mode: restricted # or "view_all"
# Database Settings
database_url: string
@@ -119,8 +127,8 @@ general_settings:
allow_requests_on_db_unavailable: boolean # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work
custom_auth: string
max_parallel_requests: 0 # the max parallel requests allowed per deployment
global_max_parallel_requests: 0 # the max parallel requests allowed on the proxy all up
max_parallel_requests: 0 # the max parallel requests allowed per deployment
global_max_parallel_requests: 0 # the max parallel requests allowed on the proxy all up
infer_model_from_keys: true
background_health_checks: true
health_check_interval: 300
@@ -138,6 +146,7 @@ router_settings:
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@@ -230,6 +239,7 @@ router_settings:
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
| store_model_in_db | boolean | If true, enables storing model + credential information in the DB. |
| supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. |
| user_mcp_management_mode | string | Controls what non-admins can see on the MCP dashboard. `restricted` (default) only lists MCP servers that the user’s teams are explicitly allowed to access. `view_all` lets every user see the full MCP server list. Tool list/call always respects per-key permissions, so users still cannot run MCP calls without access. |
| store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. |
| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
@@ -264,13 +274,14 @@ router_settings:
| forward_openai_org_id | boolean | If true, forwards the OpenAI Organization ID to the backend LLM call (if it's OpenAI). |
| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call |
| maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged |
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
### router_settings - Reference
:::info
Most values can also be set via `litellm_settings`. If you see overlapping values, settings on `router_settings` will override those on `litellm_settings`.
:::
Most values can also be set via `litellm_settings`. If you see overlapping values, settings on
`router_settings` will override those on `litellm_settings`. :::
```yaml
router_settings:
@@ -278,11 +289,12 @@ router_settings:
redis_host: <your-redis-host> # string
redis_password: <your-redis-password> # string
redis_port: <your-redis-port> # string
enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@@ -292,11 +304,11 @@ router_settings:
}
allowed_fails_policy: {
"BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
"AuthenticationErrorAllowedFails": 10, # int
"TimeoutErrorAllowedFails": 12, # int
"RateLimitErrorAllowedFails": 10000, # int
"ContentPolicyViolationErrorAllowedFails": 15, # int
"InternalServerErrorAllowedFails": 20, # int
"AuthenticationErrorAllowedFails": 10, # int
"TimeoutErrorAllowedFails": 12, # int
"RateLimitErrorAllowedFails": 10000, # int
"ContentPolicyViolationErrorAllowedFails": 15, # int
"InternalServerErrorAllowedFails": 20, # int
}
content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
@@ -312,6 +324,7 @@ router_settings:
| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
| enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) |
| tag_filtering_match_any | boolean | Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags |
| cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. |
| disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) |
| retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) |
@@ -488,6 +501,7 @@ router_settings:
| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
@@ -567,6 +581,18 @@ router_settings:
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
| FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80
| FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176
| FOCUS_PROVIDER | Destination provider for Focus exports (e.g., `s3`). Defaults to `s3`.
| FOCUS_FORMAT | Output format for Focus exports. Defaults to `parquet`.
| FOCUS_FREQUENCY | Frequency for scheduled Focus exports (`hourly`, `daily`, or `interval`). Defaults to `hourly`.
| FOCUS_CRON_OFFSET | Minute offset used when scheduling hourly/daily Focus exports. Defaults to `5` minutes.
| FOCUS_INTERVAL_SECONDS | Interval (in seconds) for Focus exports when `frequency` is `interval`.
| FOCUS_PREFIX | Object key prefix (or folder) used when uploading Focus export files. Defaults to `focus_exports`.
| FOCUS_S3_BUCKET_NAME | S3 bucket to upload Focus export files when using the S3 destination.
| FOCUS_S3_REGION_NAME | AWS region for the Focus export S3 bucket.
| FOCUS_S3_ENDPOINT_URL | Custom endpoint for the Focus export S3 client (optional; useful for S3-compatible storage).
| FOCUS_S3_ACCESS_KEY | AWS access key ID used by the Focus export S3 client.
| FOCUS_S3_SECRET_KEY | AWS secret access key used by the Focus export S3 client.
| FOCUS_S3_SESSION_TOKEN | AWS session token used by the Focus export S3 client (optional).
| FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@@ -669,6 +695,7 @@ router_settings:
| LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments
| LANGTRACE_API_KEY | API key for Langtrace service
| LASSO_API_BASE | Base URL for Lasso API
| LASSO_API_KEY | API key for Lasso service
@@ -688,6 +715,7 @@ router_settings:
| LITELLM_EMAIL | Email associated with LiteLLM account
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
| LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
@@ -707,6 +735,7 @@ router_settings:
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false"
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
@@ -774,6 +803,7 @@ router_settings:
| OTEL_EXPORTER_OTLP_HEADERS | Headers for OpenTelemetry requests
| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
| OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console)
| PAGERDUTY_API_KEY | API key for PagerDuty Alerting
| PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service
| PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service
@@ -888,4 +918,4 @@ router_settings:
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)
| ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service
| ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails
| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy
| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy
docs/my-website/docs/proxy/configs.md
+22 -1
View File
@@ -576,10 +576,31 @@ custom_tokenizer:
```yaml
general_settings:
database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20)
database_connection_pool_limit: 10 # sets connection pool per worker for prisma client to postgres db (default: 10, recommended: 10-20)
database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db
```
**How to calculate the right value:**
The connection limit is applied **per worker process**, not per instance. This means if you have multiple workers, each worker will create its own connection pool.
**Formula:**
```
database_connection_pool_limit = MAX_DB_CONNECTIONS ÷ (number_of_instances × number_of_workers_per_instance)
```
**Example:**
- Your database allows a maximum of **100 connections**
- You're running **1 instance** of LiteLLM
- Each instance has **8 workers** (set via `--num_workers 8`)
Calculation: `100 ÷ (1 × 8) = 12.5`
Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. This means:
- Each of the 8 workers will have a connection pool limit of 10
- Total maximum connections: 8 workers × 10 connections = 80 connections
- This stays safely under your database's 100 connection limit
## Extras
docs/my-website/docs/proxy/guardrails/qualifire.md
+17 -24
View File
@@ -8,13 +8,7 @@ Use [Qualifire](https://qualifire.ai) to evaluate LLM outputs for quality, safet
## Quick Start
### 1. Install the Qualifire SDK
```bash
pip install qualifire
```
### 2. Define Guardrails on your LiteLLM config.yaml
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
@@ -61,13 +55,13 @@ guardrails:
- `post_call` Run **after** LLM call, on **input & output**
- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes
### 3. Start LiteLLM Gateway
### 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 4. Test request
### 3. Test request
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
@@ -142,7 +136,7 @@ guardrails:
evaluation_id: eval_abc123 # Your evaluation ID from Qualifire dashboard
```
When `evaluation_id` is provided, LiteLLM will use `invoke_evaluation()` instead of `evaluate()`, running the pre-configured evaluation from your dashboard.
When `evaluation_id` is provided, LiteLLM will use the invoke evaluation API endpoint instead of the evaluate endpoint, running the pre-configured evaluation from your dashboard.
## Available Checks
@@ -213,19 +207,19 @@ guardrails:
### Parameter Reference
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | --------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `None` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | ---------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `https://proxy.qualifire.ai` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
### Default Behavior
@@ -261,4 +255,3 @@ This evaluates whether the LLM selected the appropriate tools and provided corre
- [Qualifire Documentation](https://docs.qualifire.ai)
- [Qualifire Dashboard](https://app.qualifire.ai)
- [Qualifire Python SDK](https://github.com/qualifire-dev/qualifire-python-sdk)
docs/my-website/docs/proxy/logging.md
-1
View File
@@ -1736,7 +1736,6 @@ class MyCustomHandler(CustomLogger):
proxy_handler_instance = MyCustomHandler()
# Set litellm.callbacks = [proxy_handler_instance] on the proxy
# need to set litellm.callbacks = [proxy_handler_instance] # on the proxy
```
#### Step 2 - Pass your custom callback class in `config.yaml`
docs/my-website/docs/proxy/prod.md
+7 -3
View File
@@ -19,7 +19,11 @@ general_settings:
master_key: sk-1234 # enter your own master key, ensure it starts with 'sk-'
alerting: ["slack"] # Setup slack alerting - get alerts on LLM exceptions, Budget Alerts, Slow LLM Responses
proxy_batch_write_at: 60 # Batch write spend updates every 60s
database_connection_pool_limit: 10 # limit the number of database connections to = MAX Number of DB Connections/Number of instances of litellm proxy (Around 10-20 is good number)
database_connection_pool_limit: 10 # connection pool limit per worker process. Total connections = limit × workers × instances. Calculate: MAX_DB_CONNECTIONS / (instances × workers). Default: 10.
:::warning
**Multiple instances:** If running multiple LiteLLM instances (e.g., Kubernetes pods), remember each instance multiplies your total connections. Example: 3 instances × 4 workers × 10 connections = 120 total connections.
:::
# OPTIONAL Best Practices
disable_error_logs: True # turn off writing LLM Exceptions to DB
@@ -54,8 +58,8 @@ For optimal performance in production, we recommend the following minimum machin
| Resource | Recommended Value |
|----------|------------------|
| CPU | 2 vCPU |
| Memory | 4 GB RAM |
| CPU | 4 vCPU |
| Memory | 8 GB RAM |
These specifications provide:
- Sufficient compute power for handling concurrent requests
docs/my-website/docs/realtime.md
+6
View File
@@ -5,6 +5,12 @@ import TabItem from '@theme/TabItem';
Use this to loadbalance across Azure + OpenAI.
Supported Providers:
- OpenAI
- Azure
- Google AI Studio (Gemini)
- Vertex AI
## Proxy Usage
### Add model to config
docs/my-website/docs/reasoning_content.md
+65
View File
@@ -591,3 +591,68 @@ Expected Response
</TabItem>
</Tabs>
## OpenAI Responses API - Auto-Summary Control
When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter.
### Enabling Auto-Summary
You can enable automatic `summary="detailed"` in two ways:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Enable auto-summary globally
litellm.reasoning_auto_summary = True
response = litellm.completion(
model="openai/responses/gpt-5-mini",
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort="low", # Will automatically add summary="detailed"
)
```
</TabItem>
<TabItem value="env" label="Environment Variable">
```bash
# Set environment variable
export LITELLM_REASONING_AUTO_SUMMARY=true
# Or in your .env file
LITELLM_REASONING_AUTO_SUMMARY=true
```
</TabItem>
<TabItem value="proxy" label="Proxy Config">
```yaml
litellm_settings:
reasoning_auto_summary: true # Enable auto-summary for all requests
model_list:
- model_name: gpt-5-mini
litellm_params:
model: openai/responses/gpt-5-mini
```
</TabItem>
</Tabs>
### Manual Control (Recommended)
For fine-grained control, pass `reasoning_effort` as a dictionary:
```python
response = litellm.completion(
model="openai/responses/gpt-5-mini",
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control
)
```
docs/my-website/docs/response_api_compact.md
+104
View File
@@ -0,0 +1,104 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# /responses/compact
Compress conversation history using OpenAI's `/responses/compact` endpoint.
| Feature | Supported |
|---------|-----------|
| Supported LiteLLM Versions | 1.72.0+ |
| Supported Providers | `openai` |
## Usage
### LiteLLM Python SDK
```python showLineNumbers title="Compact Response"
import litellm
response = litellm.compact_responses(
model="openai/gpt-4o",
input=[{"role": "user", "content": "Hello, how are you?"}],
instructions="Be helpful",
previous_response_id="resp_abc123" # optional
)
print(response.id)
print(response.object) # "response.compaction"
print(response.output)
```
### LiteLLM Proxy
<Tabs>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Compact Request"
curl http://localhost:4000/v1/responses/compact \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "openai/gpt-4o",
"input": [{"role": "user", "content": "Hello"}],
"instructions": "Be helpful"
}'
```
</TabItem>
<TabItem value="openai-sdk" label="OpenAI Python SDK">
```python showLineNumbers title="Compact with OpenAI SDK"
import httpx
response = httpx.post(
"http://localhost:4000/v1/responses/compact",
headers={"Authorization": "Bearer sk-1234"},
json={
"model": "openai/gpt-4o",
"input": [{"role": "user", "content": "Hello"}],
"instructions": "Be helpful"
}
)
print(response.json())
```
</TabItem>
</Tabs>
## Request Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use for compaction |
| `input` | string or array | Yes | Input messages to compact |
| `instructions` | string | No | System instructions |
| `previous_response_id` | string | No | ID of previous response to continue from |
## Response Format
```json
{
"id": "resp_abc123",
"object": "response.compaction",
"created_at": 1734366691,
"output": [
{
"type": "message",
"role": "assistant",
"content": [...]
},
{
"type": "compaction",
"encrypted_content": "..."
}
],
"usage": {
"input_tokens": 100,
"output_tokens": 50,
"total_tokens": 150
}
}
```
docs/my-website/img/mcp_allow_all_ui.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 135 KiB

docs/my-website/sidebars.js
+21 -39
View File
@@ -55,6 +55,7 @@ const sidebars = {
"proxy/guardrails/test_playground",
"proxy/guardrails/litellm_content_filter",
...[
"proxy/guardrails/qualifire",
"proxy/guardrails/aim_security",
"proxy/guardrails/onyx_security",
"proxy/guardrails/aporia_api",
@@ -420,14 +421,8 @@ const sidebars = {
],
},
"assistants",
{
type: "category",
label: "/audio",
items: [
"audio_transcription",
"text_to_speech",
]
},
"audio_transcription",
"text_to_speech",
{
type: "category",
label: "/batches",
@@ -477,17 +472,13 @@ const sidebars = {
"apply_guardrail",
"bedrock_invoke",
"interactions",
{
type: "category",
label: "/images",
items: [
"image_edits",
"image_generation",
"image_variations",
]
},
"image_edits",
"image_generation",
"image_variations",
"videos",
"vector_store_files",
"vector_stores/create",
"vector_stores/search",
{
type: "category",
label: "/mcp - Model Context Protocol",
@@ -531,17 +522,12 @@ const sidebars = {
"proxy/pass_through_guardrails"
]
},
{
type: "category",
label: "/rag",
items: [
"rag_ingest",
"rag_query",
]
},
"rag_ingest",
"rag_query",
"realtime",
"rerank",
"response_api",
"response_api_compact",
{
type: "category",
label: "/search",
@@ -559,14 +545,7 @@ const sidebars = {
]
},
"skills",
{
type: "category",
label: "/vector_stores",
items: [
"vector_stores/create",
"vector_stores/search",
]
},
],
},
{
@@ -675,12 +654,13 @@ const sidebars = {
"providers/bedrock_writer",
"providers/bedrock_batches",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/ai21",
"providers/aiml",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
@@ -730,7 +710,9 @@ const sidebars = {
"providers/langgraph",
"providers/lemonade",
"providers/llamafile",
"providers/llamagate",
"providers/lm_studio",
"providers/manus",
"providers/meta_llama",
"providers/milvus_vector_stores",
"providers/mistral",
flux2_test_image.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 172 KiB

litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.17.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.18.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.19.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.20.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20260106155622_add_endpoint_to_daily_activity_tables/migration.sql
+72
View File
@@ -0,0 +1,72 @@
-- DropIndex
DROP INDEX "LiteLLM_DailyAgentSpend_agent_id_date_api_key_model_custom__key";
-- DropIndex
DROP INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key";
-- DropIndex
DROP INDEX "LiteLLM_DailyOrganizationSpend_organization_id_date_api_key_key";
-- DropIndex
DROP INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key";
-- DropIndex
DROP INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key";
-- DropIndex
DROP INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key";
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN "endpoint" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN "endpoint" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN "endpoint" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN "endpoint" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN "endpoint" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "endpoint" TEXT;
-- CreateIndex
CREATE INDEX "LiteLLM_DailyAgentSpend_endpoint_idx" ON "LiteLLM_DailyAgentSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyAgentSpend_agent_id_date_api_key_model_custom__key" ON "LiteLLM_DailyAgentSpend"("agent_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_endpoint_idx" ON "LiteLLM_DailyEndUserSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key" ON "LiteLLM_DailyEndUserSpend"("end_user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyOrganizationSpend_endpoint_idx" ON "LiteLLM_DailyOrganizationSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyOrganizationSpend_organization_id_date_api_key_key" ON "LiteLLM_DailyOrganizationSpend"("organization_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyTagSpend_endpoint_idx" ON "LiteLLM_DailyTagSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key" ON "LiteLLM_DailyTagSpend"("tag", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyTeamSpend_endpoint_idx" ON "LiteLLM_DailyTeamSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyTeamSpend"("team_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyUserSpend_endpoint_idx" ON "LiteLLM_DailyUserSpend"("endpoint");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyUserSpend"("user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name", "endpoint");
litellm-proxy-extras/litellm_proxy_extras/migrations/20260107111013_add_router_settings_to_keys_teams/migration.sql
+6
View File
@@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "router_settings" JSONB DEFAULT '{}';
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "router_settings" JSONB DEFAULT '{}';
litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql
+9
View File
@@ -0,0 +1,9 @@
-- CreateIndex
-- Fixes performance issue in _check_duplicate_user_email function
-- by enabling fast case-insensitive email lookups.
--
-- Without this index, queries with mode: "insensitive" cause full table scans.
-- With this index, PostgreSQL can use an Index Scan for O(log n) performance.
--
-- Related: GitHub Issue #18411
CREATE INDEX "LiteLLM_UserTable_user_email_lower_idx" ON "LiteLLM_UserTable"(LOWER("user_email"));
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+20 -6
View File
@@ -124,6 +124,7 @@ model LiteLLM_TeamTable {
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
@@ -225,6 +226,7 @@ model LiteLLM_VerificationToken {
models String[]
aliases Json @default("{}")
config Json @default("{}")
router_settings Json? @default("{}")
user_id String?
team_id String?
permissions Json @default("{}")
@@ -422,6 +424,7 @@ model LiteLLM_DailyUserSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -433,12 +436,13 @@ model LiteLLM_DailyUserSpend {
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([user_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily organization spend metrics per model and key
@@ -451,6 +455,7 @@ model LiteLLM_DailyOrganizationSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -462,12 +467,13 @@ model LiteLLM_DailyOrganizationSpend {
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([organization_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily end user (customer) spend metrics per model and key
@@ -480,6 +486,7 @@ model LiteLLM_DailyEndUserSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -490,12 +497,13 @@ model LiteLLM_DailyEndUserSpend {
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([end_user_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily agent spend metrics per model and key
@@ -508,6 +516,7 @@ model LiteLLM_DailyAgentSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -518,12 +527,13 @@ model LiteLLM_DailyAgentSpend {
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([agent_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily team spend metrics per model and key
@@ -536,6 +546,7 @@ model LiteLLM_DailyTeamSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -547,12 +558,13 @@ model LiteLLM_DailyTeamSpend {
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([team_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily team spend metrics per model and key
@@ -566,6 +578,7 @@ model LiteLLM_DailyTagSpend {
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
@@ -577,12 +590,13 @@ model LiteLLM_DailyTagSpend {
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([tag])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.16"
version = "0.4.20"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.16"
version = "0.4.20"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+31 -1
View File
@@ -134,6 +134,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"bitbucket",
"gitlab",
"cloudzero",
"focus",
"posthog",
"levo",
]
@@ -197,6 +198,7 @@ retry = True
api_key: Optional[str] = None
openai_key: Optional[str] = None
groq_key: Optional[str] = None
gigachat_key: Optional[str] = None
databricks_key: Optional[str] = None
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
@@ -275,6 +277,7 @@ banned_keywords_list: Optional[Union[str, List]] = None
llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
guardrail_name_config_map: Dict[str, GuardrailItem] = {}
include_cost_in_streaming_usage: bool = False
reasoning_auto_summary: bool = False
### PROMPTS ####
from litellm.types.prompts.init_prompts import PromptSpec
@@ -484,6 +487,7 @@ vertex_mistral_models: Set = set()
vertex_openai_models: Set = set()
vertex_minimax_models: Set = set()
vertex_moonshot_models: Set = set()
vertex_zai_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
@@ -555,6 +559,8 @@ stability_models: Set = set()
github_copilot_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -660,6 +666,9 @@ def add_known_models():
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
key = key.replace("vertex_ai/", "")
vertex_moonshot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-zai_models":
key = key.replace("vertex_ai/", "")
vertex_zai_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.add(key)
@@ -807,6 +816,10 @@ def add_known_models():
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
elif value.get("litellm_provider") == "gigachat":
gigachat_models.add(key)
elif value.get("litellm_provider") == "llamagate":
llamagate_models.add(key)
add_known_models()
@@ -942,7 +955,8 @@ models_by_provider: dict = {
| vertex_language_models
| vertex_deepseek_models
| vertex_minimax_models
| vertex_moonshot_models,
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
@@ -1013,6 +1027,8 @@ models_by_provider: dict = {
"github_copilot": github_copilot_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,
}
# mapping for those models which have larger equivalents
@@ -1328,6 +1344,7 @@ if TYPE_CHECKING:
from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig
from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config
from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig
from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig
from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig
from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig
@@ -1357,6 +1374,7 @@ if TYPE_CHECKING:
from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig
from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig
@@ -1440,6 +1458,8 @@ if TYPE_CHECKING:
from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig
from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig
from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig as WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig
@@ -1549,6 +1569,16 @@ if TYPE_CHECKING:
# Track if async client cleanup has been registered (for lazy loading)
_async_client_cleanup_registered = False
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
# See: https://github.com/BerriAI/litellm/issues/18659
# This ensures encoding is initialized before VCR starts recording HTTP requests
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
# Load encoding at import time (pre-#18070 behavior)
# This ensures encoding is initialized before VCR starts recording
from .main import encoding
def __getattr__(name: str) -> Any:
"""Lazy import handler with cached registry for improved performance."""
litellm/_lazy_imports_registry.py
+8
View File
@@ -165,6 +165,7 @@ LLM_CONFIG_NAMES = (
"AmazonLlamaConfig",
"AmazonDeepSeekR1Config",
"AmazonMistralConfig",
"AmazonMoonshotConfig",
"AmazonTitanConfig",
"AmazonTwelveLabsPegasusConfig",
"AmazonInvokeConfig",
@@ -252,9 +253,12 @@ LLM_CONFIG_NAMES = (
"IBMWatsonXAudioTranscriptionConfig",
"GithubCopilotConfig",
"GithubCopilotResponsesAPIConfig",
"ManusResponsesAPIConfig",
"GithubCopilotEmbeddingConfig",
"NebiusConfig",
"WandbConfig",
"GigaChatConfig",
"GigaChatEmbeddingConfig",
"DashScopeChatConfig",
"MoonshotChatConfig",
"DockerModelRunnerChatConfig",
@@ -554,6 +558,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"AmazonLlamaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", "AmazonLlamaConfig"),
"AmazonDeepSeekR1Config": (".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", "AmazonDeepSeekR1Config"),
"AmazonMistralConfig": (".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", "AmazonMistralConfig"),
"AmazonMoonshotConfig": (".llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation", "AmazonMoonshotConfig"),
"AmazonTitanConfig": (".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", "AmazonTitanConfig"),
"AmazonTwelveLabsPegasusConfig": (".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", "AmazonTwelveLabsPegasusConfig"),
"AmazonInvokeConfig": (".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", "AmazonInvokeConfig"),
@@ -586,6 +591,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"),
"XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"),
"LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"),
"ManusResponsesAPIConfig": (".llms.manus.responses.transformation", "ManusResponsesAPIConfig"),
"GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"),
"OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"),
"AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"),
@@ -644,6 +650,8 @@ _LLM_CONFIGS_IMPORT_MAP = {
"GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"),
"NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
"WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
"GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"),
"GigaChatEmbeddingConfig": (".llms.gigachat.embedding.transformation", "GigaChatEmbeddingConfig"),
"DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"),
"MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"),
"DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"),
litellm/a2a_protocol/main.py
+2 -1
View File
@@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@@ -494,7 +495,7 @@ async def create_a2a_client(
async def aget_agent_card(
base_url: str,
timeout: float = 60.0,
timeout: float = DEFAULT_A2A_AGENT_TIMEOUT,
extra_headers: Optional[Dict[str, str]] = None,
) -> "AgentCard":
"""
litellm/completion_extras/litellm_responses_transformation/transformation.py
+64 -7
View File
@@ -3,6 +3,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
"""
import json
import os
from typing import (
TYPE_CHECKING,
Any,
@@ -22,6 +23,7 @@ from typing import (
from openai.types.responses.tool_param import FunctionToolParam
from pydantic import BaseModel
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -29,6 +31,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionToolParamFunctionChunk,
Reasoning,
ResponsesAPIOptionalRequestParams,
@@ -88,9 +91,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content_type = content_item.get("type")
if content_type == "output_text":
response_text = content_item.get("text", "")
# Extract annotations from content if present
annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format(
content_item.get("annotations", None)
)
msg = Message(
role=item.get("role", "assistant"),
content=response_text if response_text else "",
annotations=annotations,
)
choice = Choices(message=msg, finish_reason="stop", index=index)
return choice, index + 1
@@ -362,10 +370,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif isinstance(item, ResponseOutputMessage):
for content in item.content:
response_text = getattr(content, "text", "")
# Extract annotations from content if present
raw_annotations = getattr(content, "annotations", None)
annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format(
raw_annotations
)
msg = Message(
role=item.role,
content=response_text if response_text else "",
reasoning_content=reasoning_content,
annotations=annotations,
)
choices.append(
@@ -691,19 +705,26 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if isinstance(reasoning_effort, dict):
return Reasoning(**reasoning_effort) # type: ignore[typeddict-item]
# If string is passed, map with summary="detailed"
# Check if auto-summary is enabled via flag or environment variable
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
auto_summary_enabled = (
litellm.reasoning_auto_summary
or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
)
# If string is passed, map with optional summary based on flag/env var
if reasoning_effort == "none":
return Reasoning(effort="none", summary="detailed") # type: ignore
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none") # type: ignore
elif reasoning_effort == "high":
return Reasoning(effort="high", summary="detailed")
return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh", summary="detailed") # type: ignore[typeddict-item]
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item]
elif reasoning_effort == "medium":
return Reasoning(effort="medium", summary="detailed")
return Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
elif reasoning_effort == "low":
return Reasoning(effort="low", summary="detailed")
return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
elif reasoning_effort == "minimal":
return Reasoning(effort="minimal", summary="detailed")
return Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
return None
def _transform_response_format_to_text_format(
@@ -754,6 +775,42 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return {"format": {"type": "text"}}
return None
@staticmethod
def _convert_annotations_to_chat_format(
annotations: Optional[List[Any]],
) -> Optional[List["ChatCompletionAnnotation"]]:
"""
Convert annotations from Responses API to Chat Completions format.
Annotations are already in compatible format between both APIs,
so we just need to convert Pydantic models to dicts.
"""
if not annotations:
return None
result: List[ChatCompletionAnnotation] = []
for annotation in annotations:
try:
# Convert Pydantic models to dicts (handles both v1 and v2)
if hasattr(annotation, "model_dump"):
annotation_dict = annotation.model_dump()
elif hasattr(annotation, "dict"):
annotation_dict = annotation.dict()
elif isinstance(annotation, dict):
annotation_dict = annotation
else:
# Skip unsupported annotation types
verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}")
continue
result.append(annotation_dict) # type: ignore
except Exception as e:
# Skip malformed annotations
verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}")
continue
return result if result else None
def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str:
"""Map responses API status to chat completion finish_reason"""
litellm/constants.py
+3
View File
@@ -278,6 +278,7 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
@@ -375,6 +376,7 @@ LITELLM_CHAT_PROVIDERS = [
"perplexity",
"mistral",
"groq",
"gigachat",
"nvidia_nim",
"cerebras",
"baseten",
@@ -907,6 +909,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"twelvelabs",
"openai",
"stability",
"moonshot",
]
BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
litellm/containers/endpoint_factory.py
+2
View File
@@ -216,6 +216,8 @@ _generated_endpoints = generate_container_endpoints()
# Export generated functions dynamically
list_container_files = _generated_endpoints.get("list_container_files")
alist_container_files = _generated_endpoints.get("alist_container_files")
upload_container_file = _generated_endpoints.get("upload_container_file")
aupload_container_file = _generated_endpoints.get("aupload_container_file")
retrieve_container_file = _generated_endpoints.get("retrieve_container_file")
aretrieve_container_file = _generated_endpoints.get("aretrieve_container_file")
delete_container_file = _generated_endpoints.get("delete_container_file")
litellm/containers/endpoints.json
+10
View File
@@ -9,6 +9,16 @@
"query_params": ["after", "limit", "order"],
"response_type": "ContainerFileListResponse"
},
{
"name": "upload_container_file",
"async_name": "aupload_container_file",
"path": "/containers/{container_id}/files",
"method": "POST",
"path_params": ["container_id"],
"query_params": [],
"response_type": "ContainerFileObject",
"is_multipart": true
},
{
"name": "retrieve_container_file",
"async_name": "aretrieve_container_file",
litellm/containers/main.py
+237
View File
@@ -13,11 +13,13 @@ from litellm.main import base_llm_http_handler
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerFileListResponse,
ContainerFileObject,
ContainerListOptionalRequestParams,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
)
from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes
from litellm.utils import ProviderConfigManager, client
@@ -28,11 +30,13 @@ __all__ = [
"alist_container_files",
"alist_containers",
"aretrieve_container",
"aupload_container_file",
"create_container",
"delete_container",
"list_container_files",
"list_containers",
"retrieve_container",
"upload_container_file",
]
##### Container Create #######################
@@ -1011,3 +1015,236 @@ def list_container_files(
extra_kwargs=kwargs,
)
##### Container File Upload #######################
@client
async def aupload_container_file(
container_id: str,
file: FileTypes,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> ContainerFileObject:
"""Asynchronously upload a file to a container.
This endpoint allows uploading files directly to a container session,
supporting various file types like CSV, Excel, Python scripts, etc.
Parameters:
- `container_id` (str): The ID of the container to upload the file to
- `file` (FileTypes): The file to upload. Can be:
- A tuple of (filename, content, content_type)
- A tuple of (filename, content)
- A file-like object with read() method
- Bytes
- A string path to a file
- `timeout` (int): Request timeout in seconds
- `custom_llm_provider` (Literal["openai"]): The LLM provider to use
- `extra_headers` (Optional[Dict[str, Any]]): Additional headers
- `extra_query` (Optional[Dict[str, Any]]): Additional query parameters
- `extra_body` (Optional[Dict[str, Any]]): Additional body parameters
- `kwargs` (dict): Additional keyword arguments
Returns:
- `response` (ContainerFileObject): The uploaded file object
Example:
```python
import litellm
# Upload a CSV file
response = await litellm.aupload_container_file(
container_id="container_abc123",
file=("data.csv", open("data.csv", "rb").read(), "text/csv"),
custom_llm_provider="openai",
)
print(response)
```
"""
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["async_call"] = True
func = partial(
upload_container_file,
container_id=container_id,
file=file,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
# fmt: off
@overload
def upload_container_file(
container_id: str,
file: FileTypes,
timeout=600,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
*,
aupload_container_file: Literal[True],
**kwargs,
) -> Coroutine[Any, Any, ContainerFileObject]:
...
@overload
def upload_container_file(
container_id: str,
file: FileTypes,
timeout=600,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
*,
aupload_container_file: Literal[False] = False,
**kwargs,
) -> ContainerFileObject:
...
# fmt: on
@client
def upload_container_file(
container_id: str,
file: FileTypes,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[
ContainerFileObject,
Coroutine[Any, Any, ContainerFileObject],
]:
"""Upload a file to a container using the OpenAI Container API.
This endpoint allows uploading files directly to a container session,
supporting various file types like CSV, Excel, Python scripts, JSON, etc.
This is useful when /chat/completions or /responses sends files to the
container but the input file type is limited to PDF. This endpoint lets
you work with other file types.
Currently supports OpenAI
Example:
```python
import litellm
# Upload a CSV file
response = litellm.upload_container_file(
container_id="container_abc123",
file=("data.csv", open("data.csv", "rb").read(), "text/csv"),
custom_llm_provider="openai",
)
print(response)
# Upload a Python script
response = litellm.upload_container_file(
container_id="container_abc123",
file=("script.py", b"print('hello world')", "text/x-python"),
custom_llm_provider="openai",
)
print(response)
```
"""
from litellm.llms.custom_httpx.container_handler import generic_container_handler
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
# Check for mock response first
mock_response = kwargs.get("mock_response")
if mock_response is not None:
if isinstance(mock_response, str):
mock_response = json.loads(mock_response)
response = ContainerFileObject(**mock_response)
return response
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)
# get provider config
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
raise ValueError(f"Container provider config not found for provider: {custom_llm_provider}")
# Pre Call logging
litellm_logging_obj.update_environment_variables(
model="",
optional_params={"container_id": container_id},
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.upload_container_file.value
return generic_container_handler.handle(
endpoint_name="upload_container_file",
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
container_id=container_id,
file=file,
)
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
litellm/integrations/arize/arize.py
+2
View File
@@ -51,6 +51,7 @@ class ArizeLogger(OpenTelemetry):
space_id = os.environ.get("ARIZE_SPACE_ID")
space_key = os.environ.get("ARIZE_SPACE_KEY")
api_key = os.environ.get("ARIZE_API_KEY")
project_name = os.environ.get("ARIZE_PROJECT_NAME")
grpc_endpoint = os.environ.get("ARIZE_ENDPOINT")
http_endpoint = os.environ.get("ARIZE_HTTP_ENDPOINT")
@@ -74,6 +75,7 @@ class ArizeLogger(OpenTelemetry):
api_key=api_key,
protocol=protocol,
endpoint=endpoint,
project_name=project_name,
)
async def async_service_success_hook(
litellm/integrations/braintrust_logging.py
+29 -6
View File
@@ -225,10 +225,13 @@ class BraintrustLogger(CustomLogger):
"id": litellm_call_id,
"input": prompt["messages"],
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
@@ -351,14 +354,37 @@ class BraintrustLogger(CustomLogger):
# Allow metadata override for span name
span_name = dynamic_metadata.get("span_name", "Chat Completion")
# Span parents is a special case
span_parents = dynamic_metadata.get("span_parents")
# Convert comma-separated string to list if present
if span_parents:
span_parents = [s.strip() for s in span_parents.split(",") if s.strip()]
# Add optional span attributes only if present
span_attributes = {
"span_id": dynamic_metadata.get("span_id"),
"root_span_id": dynamic_metadata.get("root_span_id"),
"span_parents": span_parents,
}
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"output": output,
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
request_data[key] = value
if choices is not None:
request_data["output"] = [choice.dict() for choice in choices]
else:
@@ -367,9 +393,6 @@ class BraintrustLogger(CustomLogger):
if metrics is not None:
request_data["metrics"] = metrics
if metrics is not None:
request_data["metrics"] = metrics
try:
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
litellm/integrations/callback_configs.json
+6
View File
@@ -187,6 +187,12 @@
"ui_name": "Sampling Rate",
"description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)",
"required": false
},
"langsmith_tenant_id": {
"type": "text",
"ui_name": "Tenant ID",
"description": "LangSmith tenant ID for organization-scoped API keys (required when using org-scoped keys)",
"required": false
}
},
"description": "Langsmith Logging Integration"
litellm/integrations/cloudzero/database.py
+18 -164
View File
@@ -19,7 +19,7 @@
"""Database connection and data extraction for LiteLLM."""
from datetime import datetime
from typing import Any, Dict, Optional
from typing import Any, Optional, List
import polars as pl
@@ -46,19 +46,9 @@ class LiteLLMDatabase:
"""Retrieve usage data from LiteLLM daily user spend table."""
client = self._ensure_prisma_client()
# Build WHERE clause for time filtering
where_conditions = []
if start_time_utc:
where_conditions.append(f"dus.updated_at >= '{start_time_utc.isoformat()}'")
if end_time_utc:
where_conditions.append(f"dus.updated_at <= '{end_time_utc.isoformat()}'")
where_clause = ""
if where_conditions:
where_clause = "WHERE " + " AND ".join(where_conditions)
# Query to get user spend data with team information
query = f"""
# Query to get user spend data with team information. Use parameter binding to
# avoid SQL injection from user-supplied timestamps or limits.
query = """
SELECT
dus.id,
dus.date,
@@ -85,163 +75,27 @@ class LiteLLMDatabase:
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
{where_clause}
WHERE ($1::timestamptz IS NULL OR dus.updated_at >= $1::timestamptz)
AND ($2::timestamptz IS NULL OR dus.updated_at <= $2::timestamptz)
ORDER BY dus.date DESC, dus.created_at DESC
"""
if limit:
query += f" LIMIT {limit}"
params: List[Any] = [
start_time_utc,
end_time_utc,
]
if limit is not None:
try:
params.append(int(limit))
except (TypeError, ValueError):
raise ValueError("limit must be an integer")
query += " LIMIT $3"
try:
db_response = await client.db.query_raw(query)
db_response = await client.db.query_raw(query, *params)
# Convert the response to polars DataFrame with full schema inference
# This prevents schema mismatch errors when data types vary across rows
return pl.DataFrame(db_response, infer_schema_length=None)
except Exception as e:
raise Exception(f"Error retrieving usage data: {str(e)}")
async def get_table_info(self) -> Dict[str, Any]:
"""Get information about the daily user spend table."""
client = self._ensure_prisma_client()
try:
# Get row count from user spend table
user_count = await self._get_table_row_count("LiteLLM_DailyUserSpend")
# Get column structure from user spend table
query = """
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = 'LiteLLM_DailyUserSpend'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(query)
return {
"columns": columns_response,
"row_count": user_count,
"table_name": "LiteLLM_DailyUserSpend",
}
except Exception as e:
raise Exception(f"Error getting table info: {str(e)}")
async def _get_table_row_count(self, table_name: str) -> int:
"""Get row count from specified table."""
client = self._ensure_prisma_client()
try:
query = f'SELECT COUNT(*) as count FROM "{table_name}"'
response = await client.db.query_raw(query)
if response and len(response) > 0:
return response[0].get("count", 0)
return 0
except Exception:
return 0
async def discover_all_tables(self) -> Dict[str, Any]:
"""Discover all tables in the LiteLLM database and their schemas."""
client = self._ensure_prisma_client()
try:
# Get all LiteLLM tables
litellm_tables_query = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name LIKE 'LiteLLM_%'
ORDER BY table_name;
"""
tables_response = await client.db.query_raw(litellm_tables_query)
table_names = [row["table_name"] for row in tables_response]
# Get detailed schema for each table
tables_info = {}
for table_name in table_names:
# Get column information
columns_query = """
SELECT
column_name,
data_type,
is_nullable,
column_default,
character_maximum_length,
numeric_precision,
numeric_scale,
ordinal_position
FROM information_schema.columns
WHERE table_name = $1
AND table_schema = 'public'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(columns_query, table_name)
# Get primary key information
pk_query = """
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = $1::regclass AND i.indisprimary;
"""
pk_response = await client.db.query_raw(pk_query, f'"{table_name}"')
primary_keys = (
[row["attname"] for row in pk_response] if pk_response else []
)
# Get foreign key information
fk_query = """
SELECT
tc.constraint_name,
kcu.column_name,
ccu.table_name AS foreign_table_name,
ccu.column_name AS foreign_column_name
FROM information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_name = $1;
"""
fk_response = await client.db.query_raw(fk_query, table_name)
foreign_keys = fk_response if fk_response else []
# Get indexes
indexes_query = """
SELECT
i.relname AS index_name,
array_agg(a.attname ORDER BY a.attnum) AS column_names,
ix.indisunique AS is_unique
FROM pg_class t
JOIN pg_index ix ON t.oid = ix.indrelid
JOIN pg_class i ON i.oid = ix.indexrelid
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
WHERE t.relname = $1
AND t.relkind = 'r'
GROUP BY i.relname, ix.indisunique
ORDER BY i.relname;
"""
indexes_response = await client.db.query_raw(indexes_query, table_name)
indexes = indexes_response if indexes_response else []
# Get row count
try:
row_count = await self._get_table_row_count(table_name)
except Exception:
row_count = 0
tables_info[table_name] = {
"columns": columns_response,
"primary_keys": primary_keys,
"foreign_keys": foreign_keys,
"indexes": indexes,
"row_count": row_count,
}
return {
"tables": tables_info,
"table_count": len(table_names),
"table_names": table_names,
}
except Exception as e:
raise Exception(f"Error discovering tables: {str(e)}")
litellm/integrations/focus/__init__.py
View File
litellm/integrations/focus/database.py
+113
View File
@@ -0,0 +1,113 @@
"""Database access helpers for Focus export."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, Optional
import polars as pl
class FocusLiteLLMDatabase:
"""Retrieves LiteLLM usage data for Focus export workflows."""
def _ensure_prisma_client(self):
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise RuntimeError(
"Database not connected. Connect a database to your proxy - "
"https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
)
return prisma_client
async def get_usage_data(
self,
*,
limit: Optional[int] = None,
start_time_utc: Optional[datetime] = None,
end_time_utc: Optional[datetime] = None,
) -> pl.DataFrame:
"""Return usage data for the requested window."""
client = self._ensure_prisma_client()
where_clauses: list[str] = []
query_params: list[Any] = []
placeholder_index = 1
if start_time_utc:
where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz")
query_params.append(start_time_utc)
placeholder_index += 1
if end_time_utc:
where_clauses.append(f"dus.updated_at <= ${placeholder_index}::timestamptz")
query_params.append(end_time_utc)
placeholder_index += 1
where_clause = ""
if where_clauses:
where_clause = "WHERE " + " AND ".join(where_clauses)
limit_clause = ""
if limit is not None:
try:
limit_value = int(limit)
except (TypeError, ValueError) as exc: # pragma: no cover - defensive guard
raise ValueError("limit must be an integer") from exc
if limit_value < 0:
raise ValueError("limit must be non-negative")
limit_clause = f" LIMIT ${placeholder_index}"
query_params.append(limit_value)
query = f"""
SELECT
dus.id,
dus.date,
dus.user_id,
dus.api_key,
dus.model,
dus.model_group,
dus.custom_llm_provider,
dus.prompt_tokens,
dus.completion_tokens,
dus.spend,
dus.api_requests,
dus.successful_requests,
dus.failed_requests,
dus.cache_creation_input_tokens,
dus.cache_read_input_tokens,
dus.created_at,
dus.updated_at,
vt.team_id,
vt.key_alias as api_key_alias,
tt.team_alias,
ut.user_email as user_email
FROM "LiteLLM_DailyUserSpend" dus
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
{where_clause}
ORDER BY dus.date DESC, dus.created_at DESC
{limit_clause}
"""
try:
db_response = await client.db.query_raw(query, *query_params)
return pl.DataFrame(db_response, infer_schema_length=None)
except Exception as exc:
raise RuntimeError(f"Error retrieving usage data: {exc}") from exc
async def get_table_info(self) -> Dict[str, Any]:
"""Return metadata about the spend table for diagnostics."""
client = self._ensure_prisma_client()
info_query = """
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = 'LiteLLM_DailyUserSpend'
ORDER BY ordinal_position;
"""
try:
columns_response = await client.db.query_raw(info_query)
return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"}
except Exception as exc:
raise RuntimeError(f"Error getting table info: {exc}") from exc
litellm/integrations/focus/destinations/__init__.py
+12
View File
@@ -0,0 +1,12 @@
"""Destination implementations for Focus export."""
from .base import FocusDestination, FocusTimeWindow
from .factory import FocusDestinationFactory
from .s3_destination import FocusS3Destination
__all__ = [
"FocusDestination",
"FocusDestinationFactory",
"FocusTimeWindow",
"FocusS3Destination",
]
litellm/integrations/focus/destinations/base.py
+30
View File
@@ -0,0 +1,30 @@
"""Abstract destination interfaces for Focus export."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
@dataclass(frozen=True)
class FocusTimeWindow:
"""Represents the span of data exported in a single batch."""
start_time: datetime
end_time: datetime
frequency: str
class FocusDestination(Protocol):
"""Protocol for anything that can receive Focus export files."""
async def deliver(
self,
*,
content: bytes,
time_window: FocusTimeWindow,
filename: str,
) -> None:
"""Persist the serialized export for the provided time window."""
...
litellm/integrations/focus/destinations/factory.py
+59
View File
@@ -0,0 +1,59 @@
"""Factory helpers for Focus export destinations."""
from __future__ import annotations
import os
from typing import Any, Dict, Optional
from .base import FocusDestination
from .s3_destination import FocusS3Destination
class FocusDestinationFactory:
"""Builds destination instances based on provider/config settings."""
@staticmethod
def create(
*,
provider: str,
prefix: str,
config: Optional[Dict[str, Any]] = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower = provider.lower()
normalized_config = FocusDestinationFactory._resolve_config(
provider=provider_lower, overrides=config or {}
)
if provider_lower == "s3":
return FocusS3Destination(prefix=prefix, config=normalized_config)
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export"
)
@staticmethod
def _resolve_config(
*,
provider: str,
overrides: Dict[str, Any],
) -> Dict[str, Any]:
if provider == "s3":
resolved = {
"bucket_name": overrides.get("bucket_name")
or os.getenv("FOCUS_S3_BUCKET_NAME"),
"region_name": overrides.get("region_name")
or os.getenv("FOCUS_S3_REGION_NAME"),
"endpoint_url": overrides.get("endpoint_url")
or os.getenv("FOCUS_S3_ENDPOINT_URL"),
"aws_access_key_id": overrides.get("aws_access_key_id")
or os.getenv("FOCUS_S3_ACCESS_KEY"),
"aws_secret_access_key": overrides.get("aws_secret_access_key")
or os.getenv("FOCUS_S3_SECRET_KEY"),
"aws_session_token": overrides.get("aws_session_token")
or os.getenv("FOCUS_S3_SESSION_TOKEN"),
}
if not resolved.get("bucket_name"):
raise ValueError("FOCUS_S3_BUCKET_NAME must be provided for S3 exports")
return {k: v for k, v in resolved.items() if v is not None}
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export configuration"
)
litellm/integrations/focus/destinations/s3_destination.py
+74
View File
@@ -0,0 +1,74 @@
"""S3 destination implementation for Focus export."""
from __future__ import annotations
import asyncio
from datetime import timezone
from typing import Any, Optional
import boto3
from .base import FocusDestination, FocusTimeWindow
class FocusS3Destination(FocusDestination):
"""Handles uploading serialized exports to S3 buckets."""
def __init__(
self,
*,
prefix: str,
config: Optional[dict[str, Any]] = None,
) -> None:
config = config or {}
bucket_name = config.get("bucket_name")
if not bucket_name:
raise ValueError("bucket_name must be provided for S3 destination")
self.bucket_name = bucket_name
self.prefix = prefix.rstrip("/")
self.config = config
async def deliver(
self,
*,
content: bytes,
time_window: FocusTimeWindow,
filename: str,
) -> None:
object_key = self._build_object_key(time_window=time_window, filename=filename)
await asyncio.to_thread(self._upload, content, object_key)
def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str:
start_utc = time_window.start_time.astimezone(timezone.utc)
date_component = f"date={start_utc.strftime('%Y-%m-%d')}"
parts = [self.prefix, date_component]
if time_window.frequency == "hourly":
parts.append(f"hour={start_utc.strftime('%H')}")
key_prefix = "/".join(filter(None, parts))
return f"{key_prefix}/{filename}" if key_prefix else filename
def _upload(self, content: bytes, object_key: str) -> None:
client_kwargs: dict[str, Any] = {}
region_name = self.config.get("region_name")
if region_name:
client_kwargs["region_name"] = region_name
endpoint_url = self.config.get("endpoint_url")
if endpoint_url:
client_kwargs["endpoint_url"] = endpoint_url
session_kwargs: dict[str, Any] = {}
for key in (
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
):
if self.config.get(key):
session_kwargs[key] = self.config[key]
s3_client = boto3.client("s3", **client_kwargs, **session_kwargs)
s3_client.put_object(
Bucket=self.bucket_name,
Key=object_key,
Body=content,
ContentType="application/octet-stream",
)
litellm/integrations/focus/export_engine.py
+124
View File
@@ -0,0 +1,124 @@
"""Core export engine for Focus integrations (heavy dependencies)."""
from __future__ import annotations
from typing import Any, Dict, Optional
import polars as pl
from litellm._logging import verbose_logger
from .database import FocusLiteLLMDatabase
from .destinations import FocusDestinationFactory, FocusTimeWindow
from .serializers import FocusParquetSerializer, FocusSerializer
from .transformer import FocusTransformer
class FocusExportEngine:
"""Engine that fetches, normalizes, and uploads Focus exports."""
def __init__(
self,
*,
provider: str,
export_format: str,
prefix: str,
destination_config: Optional[dict[str, Any]] = None,
) -> None:
self.provider = provider
self.export_format = export_format
self.prefix = prefix
self._destination = FocusDestinationFactory.create(
provider=self.provider,
prefix=self.prefix,
config=destination_config,
)
self._serializer = self._init_serializer()
self._transformer = FocusTransformer()
self._database = FocusLiteLLMDatabase()
def _init_serializer(self) -> FocusSerializer:
if self.export_format != "parquet":
raise NotImplementedError("Only parquet export supported currently")
return FocusParquetSerializer()
async def dry_run_export_usage_data(self, limit: Optional[int]) -> Dict[str, Any]:
data = await self._database.get_usage_data(limit=limit)
normalized = self._transformer.transform(data)
usage_sample = data.head(min(50, len(data))).to_dicts()
normalized_sample = normalized.head(min(50, len(normalized))).to_dicts()
summary = {
"total_records": len(normalized),
"total_spend": self._sum_column(normalized, "spend"),
"total_tokens": self._sum_column(normalized, "total_tokens"),
"unique_teams": self._count_unique(normalized, "team_id"),
"unique_models": self._count_unique(normalized, "model"),
}
return {
"usage_data": usage_sample,
"normalized_data": normalized_sample,
"summary": summary,
}
async def export_window(
self,
*,
window: FocusTimeWindow,
limit: Optional[int],
) -> None:
data = await self._database.get_usage_data(
limit=limit,
start_time_utc=window.start_time,
end_time_utc=window.end_time,
)
if data.is_empty():
verbose_logger.debug("Focus export: no usage data for window %s", window)
return
normalized = self._transformer.transform(data)
if normalized.is_empty():
verbose_logger.debug(
"Focus export: normalized data empty for window %s", window
)
return
await self._serialize_and_upload(normalized, window)
async def _serialize_and_upload(
self, frame: pl.DataFrame, window: FocusTimeWindow
) -> None:
payload = self._serializer.serialize(frame)
if not payload:
verbose_logger.debug("Focus export: serializer returned empty payload")
return
await self._destination.deliver(
content=payload,
time_window=window,
filename=self._build_filename(),
)
def _build_filename(self) -> str:
if not self._serializer.extension:
raise ValueError("Serializer must declare a file extension")
return f"usage.{self._serializer.extension}"
@staticmethod
def _sum_column(frame: pl.DataFrame, column: str) -> float:
if frame.is_empty() or column not in frame.columns:
return 0.0
value = frame.select(pl.col(column).sum().alias("sum")).row(0)[0]
if value is None:
return 0.0
return float(value)
@staticmethod
def _count_unique(frame: pl.DataFrame, column: str) -> int:
if frame.is_empty() or column not in frame.columns:
return 0
value = frame.select(pl.col(column).n_unique().alias("unique")).row(0)[0]
if value is None:
return 0
return int(value)
litellm/integrations/focus/focus_logger.py
+211
View File
@@ -0,0 +1,211 @@
"""Focus export logger orchestrating DB pull/transform/upload."""
from __future__ import annotations
import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from .destinations import FocusTimeWindow
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from .export_engine import FocusExportEngine
else:
AsyncIOScheduler = Any
FOCUS_USAGE_DATA_JOB_NAME = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT = 500
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
def __init__(
self,
*,
provider: Optional[str] = None,
export_format: Optional[str] = None,
frequency: Optional[str] = None,
cron_offset_minute: Optional[int] = None,
interval_seconds: Optional[int] = None,
prefix: Optional[str] = None,
destination_config: Optional[dict[str, Any]] = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()
self.export_format = (
export_format or os.getenv("FOCUS_FORMAT") or "parquet"
).lower()
self.frequency = (frequency or os.getenv("FOCUS_FREQUENCY") or "hourly").lower()
self.cron_offset_minute = (
cron_offset_minute
if cron_offset_minute is not None
else int(os.getenv("FOCUS_CRON_OFFSET", "5"))
)
raw_interval = (
interval_seconds
if interval_seconds is not None
else os.getenv("FOCUS_INTERVAL_SECONDS")
)
self.interval_seconds = int(raw_interval) if raw_interval is not None else None
env_prefix = os.getenv("FOCUS_PREFIX")
self.prefix: str = (
prefix if prefix is not None else (env_prefix if env_prefix else "focus_exports")
)
self._destination_config = destination_config
self._engine: Optional["FocusExportEngine"] = None
def _ensure_engine(self) -> "FocusExportEngine":
"""Instantiate the heavy export engine lazily."""
if self._engine is None:
from .export_engine import FocusExportEngine
self._engine = FocusExportEngine(
provider=self.provider,
export_format=self.export_format,
prefix=self.prefix,
destination_config=self._destination_config,
)
return self._engine
async def export_usage_data(
self,
*,
limit: Optional[int] = None,
start_time_utc: Optional[datetime] = None,
end_time_utc: Optional[datetime] = None,
) -> None:
"""Public hook to trigger export immediately."""
if bool(start_time_utc) ^ bool(end_time_utc):
raise ValueError(
"start_time_utc and end_time_utc must be provided together"
)
if start_time_utc and end_time_utc:
window = FocusTimeWindow(
start_time=start_time_utc,
end_time=end_time_utc,
frequency=self.frequency,
)
else:
window = self._compute_time_window(datetime.now(timezone.utc))
await self._export_window(window=window, limit=limit)
async def dry_run_export_usage_data(
self, limit: Optional[int] = DEFAULT_DRY_RUN_LIMIT
) -> dict[str, Any]:
"""Return transformed data without uploading."""
engine = self._ensure_engine()
return await engine.dry_run_export_usage_data(limit=limit)
async def initialize_focus_export_job(self) -> None:
"""Entry point for scheduler jobs to run export cycle with locking."""
from litellm.proxy.proxy_server import proxy_logging_obj
pod_lock_manager = None
if proxy_logging_obj is not None:
writer = getattr(proxy_logging_obj, "db_spend_update_writer", None)
if writer is not None:
pod_lock_manager = getattr(writer, "pod_lock_manager", None)
if pod_lock_manager and pod_lock_manager.redis_cache:
acquired = await pod_lock_manager.acquire_lock(
cronjob_id=FOCUS_USAGE_DATA_JOB_NAME
)
if not acquired:
verbose_logger.debug("Focus export: unable to acquire pod lock")
return
try:
await self._run_scheduled_export()
finally:
await pod_lock_manager.release_lock(
cronjob_id=FOCUS_USAGE_DATA_JOB_NAME
)
else:
await self._run_scheduled_export()
@staticmethod
async def init_focus_export_background_job(
scheduler: AsyncIOScheduler,
) -> None:
"""Register the export cron/interval job with the provided scheduler."""
focus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=FocusLogger
)
if not focus_loggers:
verbose_logger.debug(
"No Focus export logger registered; skipping scheduler"
)
return
focus_logger = cast(FocusLogger, focus_loggers[0])
trigger_kwargs = focus_logger._build_scheduler_trigger()
scheduler.add_job(
focus_logger.initialize_focus_export_job,
**trigger_kwargs,
)
def _build_scheduler_trigger(self) -> Dict[str, Any]:
"""Return scheduler configuration for the selected frequency."""
if self.frequency == "interval":
seconds = self.interval_seconds or 60
return {"trigger": "interval", "seconds": seconds}
if self.frequency == "hourly":
minute = max(0, min(59, self.cron_offset_minute))
return {"trigger": "cron", "minute": minute, "second": 0}
if self.frequency == "daily":
total_minutes = max(0, self.cron_offset_minute)
hour = min(23, total_minutes // 60)
minute = min(59, total_minutes % 60)
return {"trigger": "cron", "hour": hour, "minute": minute, "second": 0}
raise ValueError(f"Unsupported frequency: {self.frequency}")
async def _run_scheduled_export(self) -> None:
"""Execute the scheduled export for the configured window."""
window = self._compute_time_window(datetime.now(timezone.utc))
await self._export_window(window=window, limit=None)
async def _export_window(
self,
*,
window: FocusTimeWindow,
limit: Optional[int],
) -> None:
engine = self._ensure_engine()
await engine.export_window(window=window, limit=limit)
def _compute_time_window(self, now: datetime) -> FocusTimeWindow:
"""Derive the time window to export based on configured frequency."""
now_utc = now.astimezone(timezone.utc)
if self.frequency == "hourly":
end_time = now_utc.replace(minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(hours=1)
elif self.frequency == "daily":
end_time = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=1)
elif self.frequency == "interval":
interval = timedelta(seconds=self.interval_seconds or 60)
end_time = now_utc
start_time = end_time - interval
else:
raise ValueError(f"Unsupported frequency: {self.frequency}")
return FocusTimeWindow(
start_time=start_time,
end_time=end_time,
frequency=self.frequency,
)
__all__ = ["FocusLogger"]
litellm/integrations/focus/schema.py
+50
View File
@@ -0,0 +1,50 @@
"""Schema definitions for Focus export data."""
from __future__ import annotations
import polars as pl
# see: https://focus.finops.org/focus-specification/v1-2/
FOCUS_NORMALIZED_SCHEMA = pl.Schema(
[
("BilledCost", pl.Decimal(18, 6)),
("BillingAccountId", pl.String),
("BillingAccountName", pl.String),
("BillingCurrency", pl.String),
("BillingPeriodStart", pl.Datetime(time_unit="us")),
("BillingPeriodEnd", pl.Datetime(time_unit="us")),
("ChargeCategory", pl.String),
("ChargeClass", pl.String),
("ChargeDescription", pl.String),
("ChargeFrequency", pl.String),
("ChargePeriodStart", pl.Datetime(time_unit="us")),
("ChargePeriodEnd", pl.Datetime(time_unit="us")),
("ConsumedQuantity", pl.Decimal(18, 6)),
("ConsumedUnit", pl.String),
("ContractedCost", pl.Decimal(18, 6)),
("ContractedUnitPrice", pl.Decimal(18, 6)),
("EffectiveCost", pl.Decimal(18, 6)),
("InvoiceIssuerName", pl.String),
("ListCost", pl.Decimal(18, 6)),
("ListUnitPrice", pl.Decimal(18, 6)),
("PricingCategory", pl.String),
("PricingQuantity", pl.Decimal(18, 6)),
("PricingUnit", pl.String),
("ProviderName", pl.String),
("PublisherName", pl.String),
("RegionId", pl.String),
("RegionName", pl.String),
("ResourceId", pl.String),
("ResourceName", pl.String),
("ResourceType", pl.String),
("ServiceCategory", pl.String),
("ServiceSubcategory", pl.String),
("ServiceName", pl.String),
("SubAccountId", pl.String),
("SubAccountName", pl.String),
("SubAccountType", pl.String),
("Tags", pl.Object),
]
)
__all__ = ["FOCUS_NORMALIZED_SCHEMA"]
litellm/integrations/focus/serializers/__init__.py
+6
View File
@@ -0,0 +1,6 @@
"""Serializer package exports for Focus integration."""
from .base import FocusSerializer
from .parquet import FocusParquetSerializer
__all__ = ["FocusSerializer", "FocusParquetSerializer"]
litellm/integrations/focus/serializers/base.py
+18
View File
@@ -0,0 +1,18 @@
"""Serializer abstractions for Focus export."""
from __future__ import annotations
from abc import ABC, abstractmethod
import polars as pl
class FocusSerializer(ABC):
"""Base serializer turning Focus frames into bytes."""
extension: str = ""
@abstractmethod
def serialize(self, frame: pl.DataFrame) -> bytes:
"""Convert the normalized Focus frame into the chosen format."""
raise NotImplementedError
litellm/integrations/focus/serializers/parquet.py
+22
View File
@@ -0,0 +1,22 @@
"""Parquet serializer for Focus export."""
from __future__ import annotations
import io
import polars as pl
from .base import FocusSerializer
class FocusParquetSerializer(FocusSerializer):
"""Serialize normalized Focus frames to Parquet bytes."""
extension = "parquet"
def serialize(self, frame: pl.DataFrame) -> bytes:
"""Encode the provided frame as a parquet payload."""
target = frame if not frame.is_empty() else pl.DataFrame(schema=frame.schema)
buffer = io.BytesIO()
target.write_parquet(buffer, compression="snappy")
return buffer.getvalue()
litellm/integrations/focus/transformer.py
+90
View File
@@ -0,0 +1,90 @@
"""Focus export data transformer."""
from __future__ import annotations
from datetime import timedelta
import polars as pl
from .schema import FOCUS_NORMALIZED_SCHEMA
class FocusTransformer:
"""Transforms LiteLLM DB rows into Focus-compatible schema."""
schema = FOCUS_NORMALIZED_SCHEMA
def transform(self, frame: pl.DataFrame) -> pl.DataFrame:
"""Return a normalized frame expected by downstream serializers."""
if frame.is_empty():
return pl.DataFrame(schema=self.schema)
# derive period start/end from usage date
frame = frame.with_columns(
pl.col("date")
.cast(pl.Utf8)
.str.strptime(pl.Datetime(time_unit="us"), format="%Y-%m-%d", strict=False)
.alias("usage_date"),
)
frame = frame.with_columns(
pl.col("usage_date").alias("ChargePeriodStart"),
(pl.col("usage_date") + timedelta(days=1)).alias("ChargePeriodEnd"),
)
def fmt(col):
return col.dt.strftime("%Y-%m-%dT%H:%M:%SZ")
DEC = pl.Decimal(18, 6)
def dec(col):
return col.cast(DEC)
none_str = pl.lit(None, dtype=pl.Utf8)
none_dec = pl.lit(None, dtype=pl.Decimal(18, 6))
return frame.select(
dec(pl.col("spend").fill_null(0.0)).alias("BilledCost"),
pl.col("api_key").cast(pl.String).alias("BillingAccountId"),
pl.col("api_key_alias").cast(pl.String).alias("BillingAccountName"),
pl.lit("API Key").alias("BillingAccountType"),
pl.lit("USD").alias("BillingCurrency"),
fmt(pl.col("ChargePeriodEnd")).alias("BillingPeriodEnd"),
fmt(pl.col("ChargePeriodStart")).alias("BillingPeriodStart"),
pl.lit("Usage").alias("ChargeCategory"),
none_str.alias("ChargeClass"),
pl.col("model").cast(pl.String).alias("ChargeDescription"),
pl.lit("Usage-Based").alias("ChargeFrequency"),
fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"),
fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"),
dec(pl.lit(1.0)).alias("ConsumedQuantity"),
pl.lit("Requests").alias("ConsumedUnit"),
dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"),
none_str.alias("ContractedUnitPrice"),
dec(pl.col("spend").fill_null(0.0)).alias("EffectiveCost"),
pl.col("custom_llm_provider").cast(pl.String).alias("InvoiceIssuerName"),
none_str.alias("InvoiceId"),
dec(pl.col("spend").fill_null(0.0)).alias("ListCost"),
none_dec.alias("ListUnitPrice"),
none_str.alias("AvailabilityZone"),
pl.lit("USD").alias("PricingCurrency"),
none_str.alias("PricingCategory"),
dec(pl.lit(1.0)).alias("PricingQuantity"),
none_dec.alias("PricingCurrencyContractedUnitPrice"),
dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"),
none_dec.alias("PricingCurrencyListUnitPrice"),
pl.lit("Requests").alias("PricingUnit"),
pl.col("custom_llm_provider").cast(pl.String).alias("ProviderName"),
pl.col("custom_llm_provider").cast(pl.String).alias("PublisherName"),
none_str.alias("RegionId"),
none_str.alias("RegionName"),
pl.col("model").cast(pl.String).alias("ResourceId"),
pl.col("model").cast(pl.String).alias("ResourceName"),
pl.col("model").cast(pl.String).alias("ResourceType"),
pl.lit("AI and Machine Learning").alias("ServiceCategory"),
pl.lit("Generative AI").alias("ServiceSubcategory"),
pl.col("model_group").cast(pl.String).alias("ServiceName"),
pl.col("team_id").cast(pl.String).alias("SubAccountId"),
pl.col("team_alias").cast(pl.String).alias("SubAccountName"),
none_str.alias("SubAccountType"),
none_str.alias("Tags"),
)
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
+34 -25
View File
@@ -1,28 +1,37 @@
{
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
}
}
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
},
"qualifire_eval": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.QUALIFIRE_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}"
},
"environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"]
}
}
litellm/integrations/langfuse/langfuse.py
+38 -2
View File
@@ -50,6 +50,42 @@ else:
Langfuse = Any
def _extract_cache_read_input_tokens(usage_obj) -> int:
"""
Extract cache_read_input_tokens from usage object.
Checks both:
1. Top-level cache_read_input_tokens (Anthropic format)
2. prompt_tokens_details.cached_tokens (Gemini, OpenAI format)
See: https://github.com/BerriAI/litellm/issues/18520
Args:
usage_obj: Usage object from LLM response
Returns:
int: Number of cached tokens read, defaults to 0
"""
cache_read_input_tokens = usage_obj.get("cache_read_input_tokens") or 0
# Check prompt_tokens_details.cached_tokens (used by Gemini and other providers)
if hasattr(usage_obj, "prompt_tokens_details"):
prompt_tokens_details = getattr(usage_obj, "prompt_tokens_details", None)
if (
prompt_tokens_details is not None
and hasattr(prompt_tokens_details, "cached_tokens")
):
cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None)
if (
cached_tokens is not None
and isinstance(cached_tokens, (int, float))
and cached_tokens > 0
):
cache_read_input_tokens = cached_tokens
return cache_read_input_tokens
class LangFuseLogger:
# Class variables or attributes
def __init__(
@@ -757,8 +793,8 @@ class LangFuseLogger:
cache_creation_input_tokens = (
_usage_obj.get("cache_creation_input_tokens") or 0
)
cache_read_input_tokens = (
_usage_obj.get("cache_read_input_tokens") or 0
cache_read_input_tokens = _extract_cache_read_input_tokens(
_usage_obj
)
usage = {
litellm/integrations/langsmith.py
+17 -2
View File
@@ -40,6 +40,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_sampling_rate: Optional[float] = None,
langsmith_tenant_id: Optional[str] = None,
**kwargs,
):
self.flush_lock = asyncio.Lock()
@@ -48,6 +49,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_api_key=langsmith_api_key,
langsmith_project=langsmith_project,
langsmith_base_url=langsmith_base_url,
langsmith_tenant_id=langsmith_tenant_id,
)
self.sampling_rate: float = (
langsmith_sampling_rate
@@ -76,6 +78,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_api_key: Optional[str] = None,
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_tenant_id: Optional[str] = None,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
_credentials_project = (
@@ -86,11 +89,13 @@ class LangsmithLogger(CustomBatchLogger):
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
_credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
LANGSMITH_BASE_URL=_credentials_base_url,
LANGSMITH_PROJECT=_credentials_project,
LANGSMITH_TENANT_ID=_credentials_tenant_id,
)
def _prepare_log_data(
@@ -365,8 +370,11 @@ class LangsmithLogger(CustomBatchLogger):
"""
langsmith_api_base = credentials["LANGSMITH_BASE_URL"]
langsmith_api_key = credentials["LANGSMITH_API_KEY"]
langsmith_tenant_id = credentials.get("LANGSMITH_TENANT_ID")
url = self._add_endpoint_to_url(langsmith_api_base, "runs/batch")
headers = {"x-api-key": langsmith_api_key}
if langsmith_tenant_id:
headers["x-tenant-id"] = langsmith_tenant_id
elements_to_log = [queue_object["data"] for queue_object in queue_objects]
try:
@@ -418,6 +426,7 @@ class LangsmithLogger(CustomBatchLogger):
api_key=credentials["LANGSMITH_API_KEY"],
project=credentials["LANGSMITH_PROJECT"],
base_url=credentials["LANGSMITH_BASE_URL"],
tenant_id=credentials.get("LANGSMITH_TENANT_ID"),
)
if key not in log_queue_by_credentials:
@@ -466,6 +475,9 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url=standard_callback_dynamic_params.get(
"langsmith_base_url", None
),
langsmith_tenant_id=standard_callback_dynamic_params.get(
"langsmith_tenant_id", None
),
)
else:
credentials = self.default_credentials
@@ -491,13 +503,16 @@ class LangsmithLogger(CustomBatchLogger):
def get_run_by_id(self, run_id):
langsmith_api_key = self.default_credentials["LANGSMITH_API_KEY"]
langsmith_api_base = self.default_credentials["LANGSMITH_BASE_URL"]
langsmith_tenant_id = self.default_credentials.get("LANGSMITH_TENANT_ID")
url = f"{langsmith_api_base}/runs/{run_id}"
headers = {"x-api-key": langsmith_api_key}
if langsmith_tenant_id:
headers["x-tenant-id"] = langsmith_tenant_id
response = litellm.module_level_client.get(
url=url,
headers={"x-api-key": langsmith_api_key},
headers=headers,
)
return response.json()
litellm/integrations/opentelemetry.py
+221 -115
View File
@@ -54,38 +54,6 @@ RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
def _get_litellm_resource():
"""
Create a proper OpenTelemetry Resource that respects OTEL_RESOURCE_ATTRIBUTES
while maintaining backward compatibility with LiteLLM-specific environment variables.
"""
from opentelemetry.sdk.resources import OTELResourceDetector, Resource
# Create base resource attributes with LiteLLM-specific defaults
# These will be overridden by OTEL_RESOURCE_ATTRIBUTES if present
base_attributes: Dict[str, Optional[str]] = {
"service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"),
"deployment.environment": os.getenv("OTEL_ENVIRONMENT_NAME", "production"),
# Fix the model_id to use proper environment variable or default to service name
"model_id": os.getenv(
"OTEL_MODEL_ID", os.getenv("OTEL_SERVICE_NAME", "litellm")
),
}
# Create base resource with LiteLLM-specific defaults
base_resource = Resource.create(base_attributes) # type: ignore
# Create resource from OTEL_RESOURCE_ATTRIBUTES using the detector
otel_resource_detector = OTELResourceDetector()
env_resource = otel_resource_detector.detect()
# Merge the resources: env_resource takes precedence over base_resource
# This ensures OTEL_RESOURCE_ATTRIBUTES overrides LiteLLM defaults
merged_resource = base_resource.merge(env_resource)
return merged_resource
@dataclass
class OpenTelemetryConfig:
exporter: Union[str, SpanExporter] = "console"
@@ -93,6 +61,19 @@ class OpenTelemetryConfig:
headers: Optional[str] = None
enable_metrics: bool = False
enable_events: bool = False
service_name: Optional[str] = None
deployment_environment: Optional[str] = None
model_id: Optional[str] = None
def __post_init__(self) -> None:
if not self.service_name:
self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
if not self.deployment_environment:
self.deployment_environment = os.getenv(
"OTEL_ENVIRONMENT_NAME", "production"
)
if not self.model_id:
self.model_id = os.getenv("OTEL_MODEL_ID", self.service_name)
@classmethod
def from_env(cls):
@@ -122,6 +103,9 @@ class OpenTelemetryConfig:
os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower()
== "true"
)
service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production")
model_id = os.getenv("OTEL_MODEL_ID", service_name)
if exporter == "in_memory":
return cls(exporter=InMemorySpanExporter())
@@ -131,6 +115,9 @@ class OpenTelemetryConfig:
headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
enable_metrics=enable_metrics,
enable_events=enable_events,
service_name=service_name,
deployment_environment=deployment_environment,
model_id=model_id,
)
@@ -174,6 +161,22 @@ class OpenTelemetry(CustomLogger):
self._init_logs(logger_provider)
self._init_otel_logger_on_litellm_proxy()
@staticmethod
def _get_litellm_resource(config: OpenTelemetryConfig):
"""Create an OpenTelemetry Resource using config-driven defaults."""
from opentelemetry.sdk.resources import OTELResourceDetector, Resource
base_attributes: Dict[str, Optional[str]] = {
"service.name": config.service_name,
"deployment.environment": config.deployment_environment,
"model_id": config.model_id or config.service_name,
}
base_resource = Resource.create(base_attributes) # type: ignore[arg-type]
otel_resource_detector = OTELResourceDetector()
env_resource = otel_resource_detector.detect()
return base_resource.merge(env_resource)
def _init_otel_logger_on_litellm_proxy(self):
"""
Initializes OpenTelemetry for litellm proxy server
@@ -196,50 +199,88 @@ class OpenTelemetry(CustomLogger):
litellm.service_callback.append(self)
setattr(proxy_server, "open_telemetry_logger", self)
def _get_or_create_provider(
self,
provider,
provider_name: str,
get_existing_provider_fn,
sdk_provider_class,
create_new_provider_fn,
set_provider_fn,
):
"""
Generic helper to get or create an OpenTelemetry provider (Tracer, Meter, or Logger).
Args:
provider: The provider instance passed to the init function (can be None)
provider_name: Name for logging (e.g., "TracerProvider")
get_existing_provider_fn: Function to get the existing global provider
sdk_provider_class: The SDK provider class to check for (e.g., TracerProvider from SDK)
create_new_provider_fn: Function to create a new provider instance
set_provider_fn: Function to set the provider globally
Returns:
The provider to use (either existing, new, or explicitly provided)
"""
if provider is not None:
# Provider explicitly provided (e.g., for testing)
# Do NOT call set_provider_fn - the caller is responsible for managing global state
# If they want it to be global, they've already set it before passing it to us
verbose_logger.debug(
"OpenTelemetry: Using provided TracerProvider: %s",
type(provider).__name__,
)
return provider
# Check if a provider is already set globally
try:
existing_provider = get_existing_provider_fn()
# If a real SDK provider exists (set by another SDK like Langfuse), use it
# This uses a positive check for SDK providers instead of a negative check for proxy providers
if isinstance(existing_provider, sdk_provider_class):
verbose_logger.debug(
"OpenTelemetry: Using existing %s: %s",
provider_name,
type(existing_provider).__name__,
)
provider = existing_provider
# Don't call set_provider to preserve existing context
else:
# Default proxy provider or unknown type, create our own
verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
provider = create_new_provider_fn()
set_provider_fn(provider)
except Exception as e:
# Fallback: create a new provider if something goes wrong
verbose_logger.debug(
"OpenTelemetry: Exception checking existing %s, creating new one: %s",
provider_name,
str(e),
)
provider = create_new_provider_fn()
set_provider_fn(provider)
return provider
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import SpanKind
# use provided tracer or create a new one
if tracer_provider is None:
# Check if a TracerProvider is already set globally (e.g., by Langfuse SDK)
try:
from opentelemetry.trace import ProxyTracerProvider
def create_tracer_provider():
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
provider.add_span_processor(self._get_span_processor())
return provider
existing_provider = trace.get_tracer_provider()
# If an actual provider exists (not the default proxy), use it
if not isinstance(existing_provider, ProxyTracerProvider):
verbose_logger.debug(
"OpenTelemetry: Using existing TracerProvider: %s",
type(existing_provider).__name__,
)
tracer_provider = existing_provider
# Don't call set_tracer_provider to preserve existing context
else:
# No real provider exists yet, create our own
verbose_logger.debug("OpenTelemetry: Creating new TracerProvider")
tracer_provider = TracerProvider(resource=_get_litellm_resource())
tracer_provider.add_span_processor(self._get_span_processor())
trace.set_tracer_provider(tracer_provider)
except Exception as e:
# Fallback: create a new provider if something goes wrong
verbose_logger.debug(
"OpenTelemetry: Exception checking existing provider, creating new one: %s",
str(e),
)
tracer_provider = TracerProvider(resource=_get_litellm_resource())
tracer_provider.add_span_processor(self._get_span_processor())
trace.set_tracer_provider(tracer_provider)
else:
# Tracer provider explicitly provided (e.g., for testing)
# Do NOT call set_tracer_provider - the caller is responsible for managing global state
# If they want it to be global, they've already set it before passing it to us
verbose_logger.debug(
"OpenTelemetry: Using provided TracerProvider: %s",
type(tracer_provider).__name__,
)
tracer_provider = self._get_or_create_provider(
provider=tracer_provider,
provider_name="TracerProvider",
get_existing_provider_fn=trace.get_tracer_provider,
sdk_provider_class=TracerProvider,
create_new_provider_fn=create_tracer_provider,
set_provider_fn=trace.set_tracer_provider,
)
# Grab our tracer from the TracerProvider (not from global context)
# This ensures we use the provided TracerProvider (e.g., for testing)
@@ -257,39 +298,25 @@ class OpenTelemetry(CustomLogger):
return
from opentelemetry import metrics
from opentelemetry.sdk.metrics import Histogram, MeterProvider
from opentelemetry.sdk.metrics import MeterProvider
# Only create OTLP infrastructure if no custom meter provider is provided
if meter_provider is None:
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
OTLPMetricExporter,
)
from opentelemetry.sdk.metrics.export import (
AggregationTemporality,
PeriodicExportingMetricReader,
def create_meter_provider():
metric_reader = self._get_metric_reader()
return MeterProvider(
metric_readers=[metric_reader],
resource=self._get_litellm_resource(self.config),
)
normalized_endpoint = self._normalize_otel_endpoint(
self.config.endpoint, "metrics"
)
_metric_exporter = OTLPMetricExporter(
endpoint=normalized_endpoint,
headers=OpenTelemetry._get_headers_dictionary(self.config.headers),
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
_metric_reader = PeriodicExportingMetricReader(
_metric_exporter, export_interval_millis=10000
)
meter_provider = self._get_or_create_provider(
provider=meter_provider,
provider_name="MeterProvider",
get_existing_provider_fn=metrics.get_meter_provider,
sdk_provider_class=MeterProvider,
create_new_provider_fn=create_meter_provider,
set_provider_fn=metrics.set_meter_provider,
)
meter_provider = MeterProvider(
metric_readers=[_metric_reader], resource=_get_litellm_resource()
)
meter = meter_provider.get_meter(__name__)
else:
# Use the provided meter provider as-is, without creating additional OTLP infrastructure
meter = meter_provider.get_meter(__name__)
metrics.set_meter_provider(meter_provider)
meter = meter_provider.get_meter(__name__)
self._operation_duration_histogram = meter.create_histogram(
name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38
@@ -327,22 +354,28 @@ class OpenTelemetry(CustomLogger):
if not self.config.enable_events:
return
from opentelemetry._logs import set_logger_provider
from opentelemetry._logs import get_logger_provider, set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
# set up log pipeline
if logger_provider is None:
litellm_resource = _get_litellm_resource()
logger_provider = OTLoggerProvider(resource=litellm_resource)
# Only add OTLP exporter if we created the logger provider ourselves
def create_logger_provider():
provider = OTLoggerProvider(
resource=self._get_litellm_resource(self.config)
)
log_exporter = self._get_log_exporter()
if log_exporter:
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type]
)
provider.add_log_record_processor(
BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type]
)
return provider
set_logger_provider(logger_provider)
self._get_or_create_provider(
provider=logger_provider,
provider_name="LoggerProvider",
get_existing_provider_fn=get_logger_provider,
sdk_provider_class=OTLoggerProvider,
create_new_provider_fn=create_logger_provider,
set_provider_fn=set_logger_provider,
)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
self._handle_success(kwargs, response_obj, start_time, end_time)
@@ -579,7 +612,7 @@ class OpenTelemetry(CustomLogger):
from opentelemetry.sdk.trace import TracerProvider
# Create a temporary tracer provider with dynamic headers
temp_provider = TracerProvider(resource=_get_litellm_resource())
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
temp_provider.add_span_processor(
self._get_span_processor(dynamic_headers=dynamic_headers)
)
@@ -944,6 +977,15 @@ class OpenTelemetry(CustomLogger):
if not self.config.enable_events:
return
# NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues
# with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676:
# - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal
# - LogRecord constructor no longer accepts 'resource' parameter (now inherited from LoggerProvider)
# - LogData class was removed entirely
# These logs work correctly in OTEL SDK < 1.39.0 but may fail in >= 1.39.0.
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord
@@ -951,9 +993,9 @@ class OpenTelemetry(CustomLogger):
# Get the resource from the logger provider
logger_provider = get_logger_provider()
resource = (
getattr(logger_provider, "_resource", None) or _get_litellm_resource()
)
resource = getattr(
logger_provider, "_resource", None
) or self._get_litellm_resource(self.config)
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(
@@ -1807,7 +1849,8 @@ class OpenTelemetry(CustomLogger):
)
return self.OTEL_EXPORTER
if self.OTEL_EXPORTER == "console":
otel_logs_exporter = os.getenv("OTEL_LOGS_EXPORTER")
if self.OTEL_EXPORTER == "console" or otel_logs_exporter == "console":
from opentelemetry.sdk._logs.export import ConsoleLogExporter
verbose_logger.debug(
@@ -1854,6 +1897,69 @@ class OpenTelemetry(CustomLogger):
return ConsoleLogExporter()
def _get_metric_reader(self):
"""
Get the appropriate metric reader based on the configuration.
"""
from opentelemetry.sdk.metrics import Histogram
from opentelemetry.sdk.metrics.export import (
AggregationTemporality,
ConsoleMetricExporter,
PeriodicExportingMetricReader,
)
verbose_logger.debug(
"OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
self.OTEL_EXPORTER,
self.OTEL_ENDPOINT,
self.OTEL_HEADERS,
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
normalized_endpoint = self._normalize_otel_endpoint(
self.OTEL_ENDPOINT, "metrics"
)
if self.OTEL_EXPORTER == "console":
exporter = ConsoleMetricExporter()
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
elif (
self.OTEL_EXPORTER == "otlp_http"
or self.OTEL_EXPORTER == "http/protobuf"
or self.OTEL_EXPORTER == "http/json"
):
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
exporter = OTLPMetricExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
OTLPMetricExporter,
)
exporter = OTLPMetricExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
else:
verbose_logger.warning(
"OpenTelemetry: Unknown metric exporter '%s', defaulting to console. Supported: console, otlp_http, otlp_grpc",
self.OTEL_EXPORTER,
)
exporter = ConsoleMetricExporter()
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
def _normalize_otel_endpoint(
self, endpoint: Optional[str], signal_type: str
) -> Optional[str]:
litellm/integrations/prometheus.py
+254 -10
View File
@@ -361,6 +361,25 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_requests_metric"),
)
# Cache metrics
self.litellm_cache_hits_metric = self._counter_factory(
name="litellm_cache_hits_metric",
documentation="Total number of LiteLLM cache hits",
labelnames=self.get_labels_for_metric("litellm_cache_hits_metric"),
)
self.litellm_cache_misses_metric = self._counter_factory(
name="litellm_cache_misses_metric",
documentation="Total number of LiteLLM cache misses",
labelnames=self.get_labels_for_metric("litellm_cache_misses_metric"),
)
self.litellm_cached_tokens_metric = self._counter_factory(
name="litellm_cached_tokens_metric",
documentation="Total tokens served from LiteLLM cache",
labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
@@ -823,6 +842,11 @@ class PrometheusLogger(CustomLogger):
f"standard_logging_object is required, got={standard_logging_payload}"
)
if self._should_skip_metrics_for_invalid_key(
kwargs=kwargs, standard_logging_payload=standard_logging_payload
):
return
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {})
@@ -847,7 +871,7 @@ class PrometheusLogger(CustomLogger):
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
# Include top-level metadata fields (excluding nested dictionaries)
# This allows accessing fields like requester_ip_address from top-level metadata
top_level_metadata = standard_logging_payload.get("metadata", {})
@@ -858,7 +882,7 @@ class PrometheusLogger(CustomLogger):
for k, v in top_level_metadata.items()
if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts
}
combined_metadata: Dict[str, Any] = {
**top_level_fields, # Include top-level fields first
**(_requester_metadata if _requester_metadata else {}),
@@ -977,6 +1001,12 @@ class PrometheusLogger(CustomLogger):
kwargs, start_time, end_time, enum_values, output_tokens
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
)
if (
standard_logging_payload["stream"] is True
): # log successful streaming requests from logging event hook.
@@ -1046,6 +1076,54 @@ class PrometheusLogger(CustomLogger):
standard_logging_payload["completion_tokens"]
)
def _increment_cache_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
):
"""
Increment cache-related Prometheus metrics based on cache hit/miss status.
Args:
standard_logging_payload: Contains cache_hit field (True/False/None)
enum_values: Label values for Prometheus metrics
"""
cache_hit = standard_logging_payload.get("cache_hit")
# Only track if cache_hit has a definite value (True or False)
if cache_hit is None:
return
if cache_hit is True:
# Increment cache hits counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_hits_metric"
),
enum_values=enum_values,
)
self.litellm_cache_hits_metric.labels(**_labels).inc()
# Increment cached tokens counter
total_tokens = standard_logging_payload.get("total_tokens", 0)
if total_tokens > 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cached_tokens_metric"
),
enum_values=enum_values,
)
self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens)
else:
# cache_hit is False - increment cache misses counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_misses_metric"
),
enum_values=enum_values,
)
self.litellm_cache_misses_metric.labels(**_labels).inc()
async def _increment_remaining_budget_metrics(
self,
user_api_team: Optional[str],
@@ -1237,11 +1315,17 @@ class PrometheusLogger(CustomLogger):
f"prometheus Logging - Enters failure logging function for kwargs {kwargs}"
)
# unpack kwargs
model = kwargs.get("model", "")
standard_logging_payload: StandardLoggingPayload = kwargs.get(
"standard_logging_object", {}
)
if self._should_skip_metrics_for_invalid_key(
kwargs=kwargs, standard_logging_payload=standard_logging_payload
):
return
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
@@ -1255,7 +1339,6 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias = standard_logging_payload["metadata"][
"user_api_key_team_alias"
]
kwargs.get("exception", None)
try:
self.litellm_llm_api_failed_requests_metric.labels(
@@ -1275,6 +1358,139 @@ class PrometheusLogger(CustomLogger):
pass
pass
def _extract_status_code(
self,
kwargs: Optional[dict] = None,
enum_values: Optional[Any] = None,
exception: Optional[Exception] = None,
) -> Optional[int]:
"""
Extract HTTP status code from various input formats for validation.
This is a centralized helper to extract status code from different
callback function signatures. Handles both ProxyException (uses 'code')
and standard exceptions (uses 'status_code').
Args:
kwargs: Dictionary potentially containing 'exception' key
enum_values: Object with 'status_code' attribute
exception: Exception object to extract status code from directly
Returns:
Status code as integer if found, None otherwise
"""
status_code = None
# Try from enum_values first (most common in our callbacks)
if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code:
try:
status_code = int(enum_values.status_code)
except (ValueError, TypeError):
pass
if not status_code and exception:
# ProxyException uses 'code' attribute, other exceptions may use 'status_code'
status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
if not status_code and kwargs:
exception_in_kwargs = kwargs.get("exception")
if exception_in_kwargs:
status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
return status_code
def _is_invalid_api_key_request(
self,
status_code: Optional[int],
exception: Optional[Exception] = None,
) -> bool:
"""
Determine if a request has an invalid API key based on status code and exception.
This method prevents invalid authentication attempts from being recorded in
Prometheus metrics. A 401 status code is the definitive indicator of authentication
failure. Additionally, we check exception messages for authentication error patterns
to catch cases where the exception hasn't been converted to a ProxyException yet.
Args:
status_code: HTTP status code (401 indicates authentication error)
exception: Exception object to check for auth-related error messages
Returns:
True if the request has an invalid API key and metrics should be skipped,
False otherwise
"""
if status_code == 401:
return True
# Handle cases where AssertionError is raised before conversion to ProxyException
if exception is not None:
exception_str = str(exception).lower()
auth_error_patterns = [
"virtual key expected",
"expected to start with 'sk-'",
"authentication error",
"invalid api key",
"api key not valid",
]
if any(pattern in exception_str for pattern in auth_error_patterns):
return True
return False
def _should_skip_metrics_for_invalid_key(
self,
kwargs: Optional[dict] = None,
user_api_key_dict: Optional[Any] = None,
enum_values: Optional[Any] = None,
standard_logging_payload: Optional[Union[dict, StandardLoggingPayload]] = None,
exception: Optional[Exception] = None,
) -> bool:
"""
Determine if Prometheus metrics should be skipped for invalid API key requests.
This is a centralized validation method that extracts status code and exception
information from various callback function signatures and determines if the request
represents an invalid API key attempt that should be filtered from metrics.
Args:
kwargs: Dictionary potentially containing exception and other data
user_api_key_dict: User API key authentication object (currently unused)
enum_values: Object with status_code attribute
standard_logging_payload: Standard logging payload dictionary
exception: Exception object to check directly
Returns:
True if metrics should be skipped (invalid key detected), False otherwise
"""
status_code = self._extract_status_code(
kwargs=kwargs,
enum_values=enum_values,
exception=exception,
)
if exception is None and kwargs:
exception = kwargs.get("exception")
if self._is_invalid_api_key_request(status_code, exception=exception):
verbose_logger.debug(
"Skipping Prometheus metrics for invalid API key request: "
f"status_code={status_code}, exception={type(exception).__name__ if exception else None}"
)
return True
return False
async def async_post_call_failure_hook(
self,
request_data: dict,
@@ -1300,6 +1516,14 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict,
exception=original_exception,
):
return
status_code = self._extract_status_code(exception=original_exception)
try:
_tags = StandardLoggingPayloadSetup._get_request_tags(
litellm_params=request_data,
@@ -1314,8 +1538,8 @@ class PrometheusLogger(CustomLogger):
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,
requested_model=request_data.get("model", ""),
status_code=str(getattr(original_exception, "status_code", None)),
exception_status=str(getattr(original_exception, "status_code", None)),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
tags=_tags,
route=user_api_key_dict.request_route,
@@ -1353,6 +1577,11 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict
):
return
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
hashed_api_key=user_api_key_dict.api_key,
@@ -1408,6 +1637,15 @@ class PrometheusLogger(CustomLogger):
exception = request_kwargs.get("exception", None)
llm_provider = _litellm_params.get("custom_llm_provider", None)
if self._should_skip_metrics_for_invalid_key(
kwargs=request_kwargs,
standard_logging_payload=standard_logging_payload,
):
return
hashed_api_key = standard_logging_payload.get("metadata", {}).get(
"user_api_key_hash"
)
# Create enum_values for the label factory (always create for use in different metrics)
enum_values = UserAPIKeyLabelValues(
@@ -1422,9 +1660,7 @@ class PrometheusLogger(CustomLogger):
self._get_exception_class_name(exception) if exception else None
),
requested_model=model_group,
hashed_api_key=standard_logging_payload["metadata"][
"user_api_key_hash"
],
hashed_api_key=hashed_api_key,
api_key_alias=standard_logging_payload["metadata"][
"user_api_key_alias"
],
@@ -1487,6 +1723,14 @@ class PrometheusLogger(CustomLogger):
if standard_logging_payload is None:
return
# Skip recording metrics for invalid API key requests
if self._should_skip_metrics_for_invalid_key(
kwargs=request_kwargs,
enum_values=enum_values,
standard_logging_payload=standard_logging_payload,
):
return
api_base = standard_logging_payload["api_base"]
_litellm_params = request_kwargs.get("litellm_params", {}) or {}
_metadata = _litellm_params.get("metadata", {})
litellm/litellm_core_utils/custom_logger_registry.py
+2
View File
@@ -18,6 +18,7 @@ from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLog
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.focus.focus_logger import FocusLogger
from litellm.integrations.datadog.datadog import DataDogLogger
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from litellm.integrations.deepeval import DeepEvalLogger
@@ -93,6 +94,7 @@ class CustomLoggerRegistry:
"bitbucket": BitBucketPromptManager,
"gitlab": GitLabPromptManager,
"cloudzero": CloudZeroLogger,
"focus": FocusLogger,
"posthog": PostHogLogger,
}
litellm/litellm_core_utils/get_llm_provider_logic.py
+8
View File
@@ -913,6 +913,14 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "http://localhost:2024"
)
dynamic_api_key = api_key or get_secret_str("LANGGRAPH_API_KEY")
elif custom_llm_provider == "manus":
# Manus is OpenAI compatible for responses API
api_base = (
api_base
or get_secret_str("MANUS_API_BASE")
or "https://api.manus.im"
)
dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))
litellm/litellm_core_utils/litellm_logging.py
+19 -3
View File
@@ -3630,6 +3630,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
otel_config = OpenTelemetryConfig(
exporter=arize_config.protocol,
endpoint=arize_config.endpoint,
service_name=arize_config.project_name,
)
os.environ[
@@ -3755,6 +3756,15 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
cloudzero_logger = CloudZeroLogger()
_in_memory_loggers.append(cloudzero_logger)
return cloudzero_logger # type: ignore
elif logging_integration == "focus":
from litellm.integrations.focus.focus_logger import FocusLogger
for callback in _in_memory_loggers:
if isinstance(callback, FocusLogger):
return callback # type: ignore
focus_logger = FocusLogger()
_in_memory_loggers.append(focus_logger)
return focus_logger # type: ignore
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
@@ -4075,6 +4085,12 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, CloudZeroLogger):
return callback
elif logging_integration == "focus":
from litellm.integrations.focus.focus_logger import FocusLogger
for callback in _in_memory_loggers:
if isinstance(callback, FocusLogger):
return callback
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
@@ -4799,7 +4815,7 @@ class StandardLoggingPayloadSetup:
"""
Extract additional header tags for spend tracking based on config.
"""
extra_headers: List[str] = litellm.extra_spend_tag_headers or []
extra_headers: List[str] = getattr(litellm, "extra_spend_tag_headers", None) or []
if not extra_headers:
return None
@@ -4823,9 +4839,9 @@ class StandardLoggingPayloadSetup:
metadata = litellm_params.get("metadata") or {}
litellm_metadata = litellm_params.get("litellm_metadata") or {}
if metadata.get("tags", []):
request_tags = metadata.get("tags", [])
request_tags = metadata.get("tags", []).copy()
elif litellm_metadata.get("tags", []):
request_tags = litellm_metadata.get("tags", [])
request_tags = litellm_metadata.get("tags", []).copy()
else:
request_tags = []
user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags(
litellm/litellm_core_utils/prompt_templates/common_utils.py
+12 -5
View File
@@ -6,6 +6,7 @@ import io
import mimetypes
import re
from os import PathLike
from pathlib import Path
from typing import (
TYPE_CHECKING,
Any,
@@ -533,6 +534,12 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
# Convert content to bytes
if isinstance(file_content, (str, PathLike)):
# If it's a path, open and read the file
# Extract filename from path if not already set
if filename is None:
if isinstance(file_content, PathLike):
filename = Path(file_content).name
else:
filename = Path(str(file_content)).name
with open(file_content, "rb") as f:
content = f.read()
elif isinstance(file_content, io.IOBase):
@@ -550,11 +557,11 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
# Use provided content type or guess based on filename
if not content_type:
content_type = (
mimetypes.guess_type(filename)[0]
if filename
else "application/octet-stream"
)
if filename:
guessed_type = mimetypes.guess_type(filename)[0]
content_type = guessed_type if guessed_type else "application/octet-stream"
else:
content_type = "application/octet-stream"
return ExtractedFileData(
filename=filename,
litellm/litellm_core_utils/prompt_templates/factory.py
+13
View File
@@ -2140,6 +2140,14 @@ def anthropic_messages_pt( # noqa: PLR0915
assistant_content.append(
cast(AnthropicMessagesTextParam, _cached_message)
)
# handle server_tool_use blocks (tool search, web search, etc.)
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "server_tool_use":
assistant_content.append(m) # type: ignore
# handle tool_search_tool_result blocks
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "tool_search_tool_result":
assistant_content.append(m) # type: ignore
elif (
"content" in assistant_content_block
and isinstance(assistant_content_block["content"], str)
@@ -3171,6 +3179,11 @@ def _convert_to_bedrock_tool_call_invoke(
id = tool["id"]
name = tool["function"].get("name", "")
arguments = tool["function"].get("arguments", "")
arguments_dict = json.loads(arguments) if arguments else {}
# Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object)
# When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns ""
if not isinstance(arguments_dict, dict):
arguments_dict = {}
if not arguments or not arguments.strip():
arguments_dict = {}
else:
litellm/litellm_core_utils/sensitive_data_masker.py
+54 -13
View File
@@ -1,4 +1,5 @@
from typing import Any, Dict, Optional, Set
from collections.abc import Mapping
from typing import Any, Dict, List, Optional, Set
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
@@ -17,6 +18,7 @@ class SensitiveDataMasker:
"key",
"token",
"auth",
"authorization",
"credential",
"access",
"private",
@@ -42,22 +44,52 @@ class SensitiveDataMasker:
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str, excluded_keys: Optional[Set[str]] = None) -> bool:
def is_sensitive_key(
self, key: str, excluded_keys: Optional[Set[str]] = None
) -> bool:
# Check if key is in excluded_keys first (exact match)
if excluded_keys and key in excluded_keys:
return False
key_lower = str(key).lower()
# Split on underscores and check if any segment matches the pattern
# Split on underscores/hyphens and check if any segment matches the pattern
# This avoids false positives like "max_tokens" matching "token"
# but still catches "api_key", "access_token", etc.
key_segments = key_lower.replace('-', '_').split('_')
result = any(
pattern in key_segments
for pattern in self.sensitive_patterns
)
key_segments = key_lower.replace("-", "_").split("_")
result = any(pattern in key_segments for pattern in self.sensitive_patterns)
return result
def _mask_sequence(
self,
values: List[Any],
depth: int,
max_depth: int,
excluded_keys: Optional[Set[str]],
key_is_sensitive: bool,
) -> List[Any]:
masked_items: List[Any] = []
if depth >= max_depth:
return values
for item in values:
if isinstance(item, Mapping):
masked_items.append(
self.mask_dict(dict(item), depth + 1, max_depth, excluded_keys)
)
elif isinstance(item, list):
masked_items.append(
self._mask_sequence(
item, depth + 1, max_depth, excluded_keys, key_is_sensitive
)
)
elif key_is_sensitive and isinstance(item, str):
masked_items.append(self._mask_value(item))
else:
masked_items.append(
item if isinstance(item, (int, float, bool, str, list)) else str(item)
)
return masked_items
def mask_dict(
self,
data: Dict[str, Any],
@@ -71,11 +103,20 @@ class SensitiveDataMasker:
masked_data: Dict[str, Any] = {}
for k, v in data.items():
try:
if isinstance(v, dict):
masked_data[k] = self.mask_dict(v, depth + 1, max_depth, excluded_keys)
key_is_sensitive = self.is_sensitive_key(k, excluded_keys)
if isinstance(v, Mapping):
masked_data[k] = self.mask_dict(
dict(v), depth + 1, max_depth, excluded_keys
)
elif isinstance(v, list):
masked_data[k] = self._mask_sequence(
v, depth + 1, max_depth, excluded_keys, key_is_sensitive
)
elif hasattr(v, "__dict__") and not isinstance(v, type):
masked_data[k] = self.mask_dict(vars(v), depth + 1, max_depth, excluded_keys)
elif self.is_sensitive_key(k, excluded_keys):
masked_data[k] = self.mask_dict(
vars(v), depth + 1, max_depth, excluded_keys
)
elif key_is_sensitive:
str_value = str(v) if v is not None else ""
masked_data[k] = self._mask_value(str_value)
else:
litellm/litellm_core_utils/streaming_handler.py
+43 -11
View File
@@ -2000,24 +2000,56 @@ class CustomStreamWrapper:
)
## Map to OpenAI Exception
try:
raise exception_type(
mapped_exception = exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
except Exception as e:
from litellm.exceptions import MidStreamFallbackError
except Exception as mapping_error:
mapped_exception = mapping_error
raise MidStreamFallbackError(
message=str(e),
model=self.model,
llm_provider=self.custom_llm_provider or "anthropic",
original_exception=e,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
)
def _normalize_status_code(exc: Exception) -> Optional[int]:
"""
Best-effort status_code extraction.
Uses status_code on the exception, then falls back to the response.
"""
try:
code = getattr(exc, "status_code", None)
if code is not None:
return int(code)
except Exception:
pass
response = getattr(exc, "response", None)
if response is not None:
try:
status_code = getattr(response, "status_code", None)
if status_code is not None:
return int(status_code)
except Exception:
pass
return None
mapped_status_code = _normalize_status_code(mapped_exception)
original_status_code = _normalize_status_code(e)
if mapped_status_code is not None and 400 <= mapped_status_code < 500:
raise mapped_exception
if original_status_code is not None and 400 <= original_status_code < 500:
raise mapped_exception
from litellm.exceptions import MidStreamFallbackError
raise MidStreamFallbackError(
message=str(mapped_exception),
model=self.model,
llm_provider=self.custom_llm_provider or "anthropic",
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
)
@staticmethod
def _strip_sse_data_from_chunk(chunk: Optional[str]) -> Optional[str]:
litellm/llms/anthropic/chat/transformation.py
+8 -7
View File
@@ -1265,14 +1265,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_creation_tokens=cache_creation_input_tokens,
cache_creation_token_details=cache_creation_token_details,
)
completion_token_details = (
CompletionTokensDetailsWrapper(
reasoning_tokens=token_counter(
text=reasoning_content, count_response_tokens=True
)
)
# Always populate completion_token_details, not just when there's reasoning_content
reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else None
else 0
)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
text_tokens=completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens,
)
total_tokens = prompt_tokens + completion_tokens
litellm/llms/azure/azure.py
+13
View File
@@ -990,6 +990,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
def create_azure_base_url(
self, azure_client_params: dict, model: Optional[str]
) -> str:
from litellm.llms.azure_ai.image_generation import (
AzureFoundryFluxImageGenerationConfig,
)
api_base: str = azure_client_params.get(
"azure_endpoint", ""
) # "https://example-endpoint.openai.azure.com"
@@ -999,6 +1003,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if model is None:
model = ""
# Handle FLUX 2 models on Azure AI which use a different URL pattern
# e.g., /providers/blackforestlabs/v1/flux-2-pro instead of /openai/deployments/{model}/images/generations
if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model):
return AzureFoundryFluxImageGenerationConfig.get_flux2_image_generation_url(
api_base=api_base,
model=model,
api_version=api_version,
)
if "/openai/deployments/" in api_base:
base_url_with_deployment = api_base
else:
litellm/llms/azure_ai/image_edit/__init__.py
+20 -7
View File
@@ -1,15 +1,28 @@
from litellm.llms.azure_ai.image_generation.flux_transformation import (
AzureFoundryFluxImageGenerationConfig,
)
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .flux2_transformation import AzureFoundryFlux2ImageEditConfig
from .transformation import AzureFoundryFluxImageEditConfig
__all__ = ["AzureFoundryFluxImageEditConfig"]
__all__ = ["AzureFoundryFluxImageEditConfig", "AzureFoundryFlux2ImageEditConfig"]
def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig:
model = model.lower()
model = model.replace("-", "")
model = model.replace("_", "")
if model == "" or "flux" in model: # empty model is flux
"""
Get the appropriate image edit config for an Azure AI model.
- FLUX 2 models use JSON with base64 image
- FLUX 1 models use multipart/form-data
"""
# Check if it's a FLUX 2 model
if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model):
return AzureFoundryFlux2ImageEditConfig()
# Default to FLUX 1 config for other FLUX models
model_normalized = model.lower().replace("-", "").replace("_", "")
if model_normalized == "" or "flux" in model_normalized:
return AzureFoundryFluxImageEditConfig()
else:
raise ValueError(f"Model {model} is not supported for Azure AI image editing.")
raise ValueError(f"Model {model} is not supported for Azure AI image editing.")
litellm/llms/azure_ai/image_edit/flux2_transformation.py
+167
View File
@@ -0,0 +1,167 @@
import base64
from io import BufferedReader
from typing import Any, Dict, Optional, Tuple
from httpx._types import RequestFiles
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.image_generation.flux_transformation import (
AzureFoundryFluxImageGenerationConfig,
)
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
"""
Azure AI Foundry FLUX 2 image edit config
Supports FLUX 2 models (e.g., flux.2-pro) for image editing.
Uses the same /providers/blackforestlabs/v1/flux-2-pro endpoint as image generation,
with the image passed as base64 in JSON body.
"""
def get_supported_openai_params(self, model: str) -> list:
"""
FLUX 2 supports a subset of OpenAI image edit params
"""
return [
"prompt",
"image",
"model",
"n",
"size",
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""
Map OpenAI params to FLUX 2 params.
FLUX 2 uses the same param names as OpenAI for supported params.
"""
mapped_params: Dict[str, Any] = {}
supported_params = self.get_supported_openai_params(model)
for key, value in dict(image_edit_optional_params).items():
if key in supported_params and value is not None:
mapped_params[key] = value
return mapped_params
def use_multipart_form_data(self) -> bool:
"""FLUX 2 uses JSON requests, not multipart/form-data."""
return False
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
"""
Validate Azure AI Foundry environment and set up authentication
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key,
"Content-Type": "application/json",
}
)
return headers
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
"""
Transform image edit request for FLUX 2.
FLUX 2 uses the same endpoint for generation and editing,
with the image passed as base64 in the JSON body.
"""
image_b64 = self._convert_image_to_base64(image)
# Build request body with required params
request_body: Dict[str, Any] = {
"prompt": prompt,
"image": image_b64,
"model": model,
}
# Add mapped optional params (already filtered by map_openai_params)
request_body.update(image_edit_optional_request_params)
# Return JSON body and empty files list (FLUX 2 doesn't use multipart)
return request_body, []
def _convert_image_to_base64(self, image: Any) -> str:
"""Convert image file to base64 string"""
# Handle list of images (take first one)
if isinstance(image, list):
if len(image) == 0:
raise ValueError("Empty image list provided")
image = image[0]
if isinstance(image, BufferedReader):
image_bytes = image.read()
image.seek(0) # Reset file pointer for potential reuse
elif isinstance(image, bytes):
image_bytes = image
elif hasattr(image, "read"):
image_bytes = image.read() # type: ignore
else:
raise ValueError(f"Unsupported image type: {type(image)}")
return base64.b64encode(image_bytes).decode("utf-8")
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Constructs a complete URL for Azure AI Foundry FLUX 2 image edits.
Uses the same /providers/blackforestlabs/v1/flux-2-pro endpoint as image generation.
"""
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter."
)
api_version = (
litellm_params.get("api_version")
or litellm.api_version
or get_secret_str("AZURE_AI_API_VERSION")
or "preview"
)
return AzureFoundryFluxImageGenerationConfig.get_flux2_image_generation_url(
api_base=api_base,
model=model,
api_version=api_version,
)
litellm/llms/azure_ai/image_edit/transformation.py
+5 -3
View File
@@ -71,9 +71,11 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
"Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter."
)
api_version = (litellm_params.get("api_version") or litellm.api_version
or get_secret_str("AZURE_AI_API_VERSION")
)
api_version = (
litellm_params.get("api_version")
or litellm.api_version
or get_secret_str("AZURE_AI_API_VERSION")
)
if api_version is None:
# API version is mandatory for Azure AI Foundry
raise ValueError(
litellm/llms/azure_ai/image_generation/flux_transformation.py
+55 -1
View File
@@ -1,3 +1,5 @@
from typing import Optional
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
@@ -11,4 +13,56 @@ class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig):
From our test suite - following GPTImageGenerationConfig is working for this model
"""
pass
@staticmethod
def get_flux2_image_generation_url(
api_base: Optional[str],
model: str,
api_version: Optional[str],
) -> str:
"""
Constructs the complete URL for Azure AI FLUX 2 image generation.
FLUX 2 models on Azure AI use a different URL pattern than standard Azure OpenAI:
- Standard: /openai/deployments/{model}/images/generations
- FLUX 2: /providers/blackforestlabs/v1/flux-2-pro
Args:
api_base: Base URL (e.g., https://litellm-ci-cd-prod.services.ai.azure.com)
model: Model name (e.g., flux.2-pro)
api_version: API version (e.g., preview)
Returns:
Complete URL for the FLUX 2 image generation endpoint
"""
if api_base is None:
raise ValueError(
"api_base is required for Azure AI FLUX 2 image generation"
)
api_base = api_base.rstrip("/")
api_version = api_version or "preview"
# If the api_base already contains /providers/, it's already a complete path
if "/providers/" in api_base:
if "?" in api_base:
return api_base
return f"{api_base}?api-version={api_version}"
# Construct the FLUX 2 provider path
# Model name flux.2-pro maps to endpoint flux-2-pro
return f"{api_base}/providers/blackforestlabs/v1/flux-2-pro?api-version={api_version}"
@staticmethod
def is_flux2_model(model: str) -> bool:
"""
Check if the model is an Azure AI FLUX 2 model.
Args:
model: Model name (e.g., flux.2-pro, azure_ai/flux.2-pro)
Returns:
True if the model is a FLUX 2 model
"""
model_lower = model.lower().replace(".", "-").replace("_", "-")
return "flux-2" in model_lower or "flux2" in model_lower
litellm/llms/base_llm/responses/transformation.py
+27
View File
@@ -242,3 +242,30 @@ class BaseResponsesAPIConfig(ABC):
#########################################################
########## END CANCEL RESPONSE API TRANSFORMATION #######
#########################################################
#########################################################
########## COMPACT RESPONSE API TRANSFORMATION ##########
#########################################################
@abstractmethod
def transform_compact_response_api_request(
self,
model: str,
input: Union[str, ResponseInputParam],
response_api_optional_request_params: Dict,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
pass
@abstractmethod
def transform_compact_response_api_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
pass
#########################################################
########## END COMPACT RESPONSE API TRANSFORMATION ######
#########################################################
litellm/llms/bedrock/base_aws_llm.py
+4
View File
@@ -369,6 +369,10 @@ class BaseAWSLLM:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="stability"
)
elif provider == "moonshot" and "moonshot/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="moonshot"
)
return model_id
@staticmethod

Some files were not shown because too many files have changed in this diff Show More