tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-28 22:20:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into litellm_deleted_keys_team

Browse Source
This commit is contained in:
yuneng-jiang
2026-01-15 15:55:18 -08:00
parent 2a4f883e78 6a7edd8f2b
commit 5ff768e5e0
1305 changed files with 107472 additions and 15170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+259 -82
View File
@@ -178,6 +178,7 @@ jobs:
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
pip install "websockets==13.1.0"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
@@ -208,7 +209,10 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
# Add --timeout to kill hanging tests after 300s (5 min)
# Add -v to show test names as they run for debugging
# Add --tb=short for shorter tracebacks
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=20 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4 --timeout=300 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -614,6 +618,12 @@ jobs:
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
python --version
which python
pip install --upgrade typing-extensions>=4.12.0
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
@@ -657,7 +667,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@@ -677,6 +687,9 @@ jobs:
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
set +e
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
@@ -685,6 +698,9 @@ jobs:
- run:
name: Run tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pwd
ls
python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
@@ -1090,13 +1106,16 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=5 -n 4
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1446,7 +1465,7 @@ jobs:
- run:
name: Run core tests
command: |
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1460,6 +1479,60 @@ jobs:
paths:
- litellm_core_tests_coverage.xml
- litellm_core_tests_coverage
litellm_mapped_tests_litellm_core_utils:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run litellm_core_utils tests
command: |
python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_core_utils_tests_coverage.xml
mv .coverage litellm_core_utils_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_core_utils_tests_coverage.xml
- litellm_core_utils_tests_coverage
litellm_mapped_tests_integrations:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run integrations tests
command: |
python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_integrations_tests_coverage.xml
mv .coverage litellm_integrations_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_integrations_tests_coverage.xml
- litellm_integrations_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
@@ -1886,6 +1959,18 @@ jobs:
command: |
kind create cluster --name litellm-test
- run:
name: Build Docker image for helm tests
command: |
IMAGE_TAG=${CIRCLE_SHA1:-ci}
docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database .
- run:
name: Load Docker image into Kind
command: |
IMAGE_TAG=${CIRCLE_SHA1:-ci}
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
# Run helm lint
- run:
name: Run helm lint
@@ -1896,7 +1981,11 @@ jobs:
- run:
name: Run helm tests
command: |
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml
IMAGE_TAG=${CIRCLE_SHA1:-ci}
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
--set image.repository=litellm-ci \
--set image.tag=${IMAGE_TAG} \
--set image.pullPolicy=Never
# Wait for pod to be ready
echo "Waiting 30 seconds for pod to be ready..."
sleep 30
@@ -1941,11 +2030,13 @@ jobs:
- run: ruff check ./litellm
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/check_provider_folders_documented.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
- run: python ./tests/code_coverage_tests/test_proxy_types_import.py
- run: python ./tests/code_coverage_tests/callback_manager_test.py
- run: python ./tests/code_coverage_tests/recursive_detector.py
@@ -1961,6 +2052,7 @@ jobs:
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@@ -1989,10 +2081,13 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install apscheduler
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
name: Load Docker Database Image
command: |
docker build -t myapp . -f ./docker/Dockerfile.database
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@@ -2005,7 +2100,7 @@ jobs:
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
--name my-app \
myapp:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
@@ -2024,10 +2119,11 @@ jobs:
name: Check container logs for expected message
command: |
echo "=== Printing Full Container Startup Logs ==="
docker logs my-app
LOG_OUTPUT="$(docker logs my-app 2>&1)"
printf '%s\n' "$LOG_OUTPUT"
echo "=== End of Full Container Startup Logs ==="
if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
echo "Expected message found in logs. Test passed."
else
echo "Expected message not found in logs. Test failed."
@@ -2108,7 +2204,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@@ -2250,16 +2346,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@@ -2294,7 +2394,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -2390,16 +2490,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -2432,7 +2536,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -2483,7 +2587,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@@ -2551,16 +2655,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -2584,7 +2692,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -2664,16 +2772,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container 1
# intentionally give bad redis credentials here
@@ -2693,7 +2805,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -2714,7 +2826,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4001 \
--detailed_debug
@@ -2800,16 +2912,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -2824,7 +2940,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -3032,17 +3148,20 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
# Run pytest and generate JUnit XML report
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@@ -3064,7 +3183,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@@ -3402,6 +3521,37 @@ jobs:
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
build_docker_database_image:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- run:
name: Upgrade Docker
command: |
curl -fsSL https://get.docker.com | sh
docker version
- run:
name: Build Docker image
command: |
docker build \
-t litellm-docker-database:ci \
-f docker/Dockerfile.database .
- run:
name: Save Docker image to workspace root
command: |
docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz
- persist_to_workspace:
root: .
paths:
- litellm-docker-database.tar.gz
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
@@ -3413,68 +3563,54 @@ jobs:
- attach_workspace:
at: ~/project
- run:
name: Upgrade Docker to v24.x (API 1.44+)
name: Load Docker Database Image
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda activate myenv
python --version
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
command: |
npm install -D @playwright/test
npm install @google-cloud/vertexai
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
- run:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7} \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7} \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$SMALL_DATABASE_URL \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name my-app \
--name litellm-docker-database \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@@ -3488,7 +3624,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
command: docker logs -f litellm-docker-database
background: true
- run:
name: Wait for app to be ready
@@ -3496,7 +3632,10 @@ jobs:
- run:
name: Run Playwright Tests
command: |
npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
npx playwright test \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 120m
- store_artifacts:
path: test-results
@@ -3549,7 +3688,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@@ -3686,9 +3825,17 @@ workflows:
only:
- main
- /litellm_.*/
- build_docker_database_image:
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
context: e2e_ui_tests
requires:
- ui_build
- build_docker_database_image
filters:
branches:
only:
@@ -3701,30 +3848,40 @@ workflows:
- main
- /litellm_.*/
- e2e_openai_endpoints:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_store_model_in_db_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@@ -3737,6 +3894,8 @@ workflows:
- main
- /litellm_.*/
- proxy_pass_through_endpoint_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@@ -3808,6 +3967,18 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_integrations:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_litellm_core_utils:
filters:
branches:
only:
- main
- /litellm_.*/
- batches_testing:
filters:
branches:
@@ -3856,6 +4027,8 @@ workflows:
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
@@ -3875,6 +4048,8 @@ workflows:
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image
filters:
branches:
only:
@@ -3925,6 +4100,8 @@ workflows:
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
.gitguardian.yaml
+27
View File
@@ -64,6 +64,30 @@ secret:
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
- name: Test Base64 Basic Auth header in pass_through_endpoints test
match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
# PostgreSQL password "postgres" in CI configs - standard test database password
- name: Test PostgreSQL password in CI configurations
match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
# Bearer token in locustfile.py - test/example API key for load testing
- name: Test Bearer token in locustfile load test
match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
# Inkeep API key in docusaurus.config.js - public documentation site key
- name: Inkeep API key in documentation config
match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
# Langfuse credentials in test_completion.py - test credentials for integration test
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
@@ -82,3 +106,6 @@ secret:
- name: Test API key patterns
match: test-api-key
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b
.github/ISSUE_TEMPLATE/bug_report.yml
+18
View File
@@ -7,6 +7,8 @@ body:
attributes:
value: |
Thanks for taking the time to fill out this bug report!
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
- type: textarea
id: what-happened
attributes:
@@ -16,6 +18,21 @@ body:
value: "A bug happened!"
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to Reproduce
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
placeholder: |
1. config.yaml file/ .env file/ etc.
2. Run the following code...
3. Observe the error...
value: |
1.
2.
3.
validations:
required: true
- type: textarea
id: logs
attributes:
@@ -27,6 +44,7 @@ body:
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
.github/ISSUE_TEMPLATE/feature_request.yml
+1
View File
@@ -27,6 +27,7 @@ body:
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
.github/workflows/ghcr_deploy.yml
+22 -7
View File
@@ -5,6 +5,7 @@ on:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
@@ -336,9 +337,9 @@ jobs:
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
else
# Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
# Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
fi
@@ -350,28 +351,42 @@ jobs:
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
version-fragment: 'bug'
# Add suffix for non-stable releases (semantic versioning)
- name: Calculate chart version with prerelease suffix
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
INPUT_TAG="${{ github.event.inputs.tag }}"
# Chart version (independent Helm chart versioning with release type suffix)
if [ "$RELEASE_TYPE" = "stable" ]; then
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
else
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
fi
# App version (must match Docker tags)
# stable/rc releases: Docker creates main-{tag}, so use the tag
# latest/dev releases: Docker only creates main-{release_type}, so use release_type
if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
APP_VERSION="${INPUT_TAG}"
else
APP_VERSION="${RELEASE_TYPE}"
fi
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}
.github/workflows/label-component.yml
+56 -118
View File
@@ -11,134 +11,72 @@ jobs:
permissions:
issues: write
steps:
- name: Add SDK label
if: contains(github.event.issue.body, 'SDK (litellm Python package)')
- name: Add component labels
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'sdk';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
const body = context.payload.issue.body;
if (!body) return;
- name: Add Proxy label
if: contains(github.event.issue.body, 'Proxy')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'proxy';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
});
} else {
throw error;
// Define component mappings with regex patterns that handle flexible whitespace
const components = [
{
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
label: 'sdk',
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
},
{
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
label: 'proxy',
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
},
{
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
label: 'ui-dashboard',
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
},
{
pattern: /What part of LiteLLM is this about\?\s*Docs/,
label: 'docs',
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
];
- name: Add UI Dashboard label
if: contains(github.event.issue.body, 'UI Dashboard')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'ui-dashboard';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
// Find matching component
for (const component of components) {
if (component.pattern.test(body)) {
// Ensure label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label,
color: component.color,
description: component.description
});
}
}
- name: Add Docs label
if: contains(github.event.issue.body, 'Docs')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'docs';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
// Add label to issue
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
issue_number: context.issue.number,
labels: [component.label]
});
} else {
throw error;
break;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
.github/workflows/locustfile.py
+1 -1
View File
@@ -8,7 +8,7 @@ class MyUser(HttpUser):
def chat_completion(self):
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer sk-test-load-test-key-123",
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
# Include any additional headers you may need for authentication, etc.
}
.github/workflows/publish-migrations.yml
+3 -2
View File
@@ -13,6 +13,7 @@ on:
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:
@@ -20,7 +21,7 @@ jobs:
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: test-postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
@@ -35,7 +36,7 @@ jobs:
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: test-postgres
POSTGRES_PASSWORD: postgres
ports:
- 5433:5432
options: >-
.github/workflows/test-litellm.yml
+1
View File
@@ -35,6 +35,7 @@ jobs:
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
.gitignore
+6
View File
@@ -59,6 +59,7 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/_next/
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
@@ -100,3 +101,8 @@ update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage
AGENTS.md
+21
View File
@@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that:
- Test provider-specific functionality thoroughly
- Consider adding load tests for performance-critical changes
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
2. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
- **Structure tests properly**:
- First test should verify the component renders successfully
- Subsequent tests should focus on functionality and user interactions
- Use `waitFor` for async operations that aren't already awaited
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
### IMPORTANT PATTERNS
1. **Function/Tool Calling**:
ARCHITECTURE.md
+277
View File
@@ -0,0 +1,277 @@
# LiteLLM Architecture - LiteLLM SDK + AI Gateway
This document helps contributors understand where to make changes in LiteLLM.
---
## How It Works
The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
```
OpenAI SDK (client) ──▶ LiteLLM AI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Anthropic SDK (client) ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Any HTTP client ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
```
The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
---
## 1. AI Gateway (Proxy) Request Flow
The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
```mermaid
sequenceDiagram
participant Client
participant ProxyServer as proxy/proxy_server.py
participant Auth as proxy/auth/user_api_key_auth.py
participant Hooks as proxy/hooks/
participant Router as router.py
participant Main as main.py
participant Handler as llms/custom_httpx/llm_http_handler.py
participant Transform as llms/{provider}/chat/transformation.py
participant Provider as LLM Provider API
Client->>ProxyServer: POST /v1/chat/completions
ProxyServer->>Auth: user_api_key_auth()
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
Main->>Handler: BaseLLMHTTPHandler.completion()
Handler->>Transform: ProviderConfig.transform_request()
Handler->>Provider: HTTP Request
Provider-->>Handler: Response
Handler->>Transform: ProviderConfig.transform_response()
Handler-->>Hooks: async_log_success_event()
Handler-->>Client: ModelResponse
```
### Proxy Components
```mermaid
graph TD
subgraph "Incoming Request"
Client["POST /v1/chat/completions"]
end
subgraph "proxy/proxy_server.py"
Endpoint["chat_completion()"]
end
subgraph "proxy/auth/"
Auth["user_api_key_auth()"]
end
subgraph "proxy/"
PreCall["litellm_pre_call_utils.py"]
RouteRequest["route_llm_request.py"]
end
subgraph "litellm/"
Router["router.py"]
Main["main.py"]
end
Client --> Endpoint
Endpoint --> Auth
Auth --> PreCall
PreCall --> RouteRequest
RouteRequest --> Router
Router --> Main
Main --> Client
```
**Key proxy files:**
- `proxy/proxy_server.py` - Main API endpoints
- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
- `proxy/hooks/` - Proxy-level callbacks
- `router.py` - Load balancing, fallbacks
- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
**LLM-specific proxy endpoints:**
| Endpoint | Directory | Purpose |
|----------|-----------|---------|
| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
**Proxy Hooks** (`proxy/hooks/__init__.py`):
| Hook | File | Purpose |
|------|------|---------|
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
---
## 2. SDK Request Flow
The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
```mermaid
graph TD
subgraph "SDK Entry Points"
Completion["litellm.completion()"]
Messages["litellm.messages()"]
end
subgraph "main.py"
Main["completion()<br/>acompletion()"]
end
subgraph "utils.py"
GetProvider["get_llm_provider()"]
end
subgraph "llms/custom_httpx/"
Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
end
subgraph "llms/{provider}/chat/"
TransformReq["transform_request()"]
TransformResp["transform_response()"]
end
subgraph "litellm_core_utils/"
Streaming["streaming_handler.py"]
end
subgraph "integrations/ (async, off main thread)"
Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
end
Completion --> Main
Messages --> Main
Main --> GetProvider
GetProvider --> Handler
Handler --> TransformReq
TransformReq --> HTTP
HTTP --> Provider["LLM Provider API"]
Provider --> HTTP
HTTP --> TransformResp
TransformResp --> Streaming
Streaming --> Response["ModelResponse"]
Response -.->|async| Callbacks
```
**Key SDK files:**
- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
- `utils.py` - `get_llm_provider()` resolves model → provider
- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
- `litellm_core_utils/streaming_handler.py` - Streaming response handling
- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
---
## 3. Translation Layer
When a request comes in, it goes through a **translation layer** that converts between API formats.
Each translation is isolated in its own file, making it easy to test and modify independently.
### Where to find translations
| Incoming API | Provider | Translation File |
|--------------|----------|------------------|
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
### Example: Debugging prompt caching
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
3. Compare how each handles `cache_control` in `transform_request()`
### How translations work
Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
```python
class ProviderConfig(BaseConfig):
def transform_request(self, model, messages, optional_params, litellm_params, headers):
# Convert OpenAI format → Provider format
return {"messages": transformed_messages, ...}
def transform_response(self, model, raw_response, model_response, logging_obj, ...):
# Convert Provider format → OpenAI format
return ModelResponse(choices=[...], usage=Usage(...))
```
The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
---
## 4. Adding/Modifying Providers
### To add a new provider:
1. Create `llms/{provider}/chat/transformation.py`
2. Implement `Config` class with `transform_request()` and `transform_response()`
3. Add tests in `tests/llm_translation/test_{provider}.py`
### To add a feature (e.g., prompt caching):
1. Find the translation file from the table above
2. Modify `transform_request()` to handle the new parameter
3. Add unit tests that verify the transformation
### Testing checklist
When adding a feature, verify it works across all paths:
| Test | File Pattern |
|------|--------------|
| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
| Vertex AI | `tests/llm_translation/test_vertex*.py` |
| Gemini | `tests/llm_translation/test_gemini*.py` |
### Unit testing translations
Translations are designed to be unit testable without making API calls:
```python
from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
def test_prompt_caching_transform():
config = BedrockConverseConfig()
result = config.transform_request(
model="anthropic.claude-3-opus",
messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
optional_params={},
litellm_params={},
headers={}
)
assert "cachePoint" in str(result) # Verify cache_control was translated
```
Dockerfile
+7 -4
View File
@@ -20,7 +20,8 @@ RUN python -m pip install build
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -65,12 +66,14 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
Makefile
+2 -1
View File
@@ -45,6 +45,7 @@ install-proxy-dev-ci:
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install openapi-core
cd enterprise && poetry run pip install -e . && cd ..
install-helm-unittest:
@@ -100,4 +101,4 @@ test-llm-translation-single: install-test-deps
@mkdir -p test-results
poetry run pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
-v --tb=short --maxfail=100 --timeout=300
README.md
+1 -1
View File
@@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
@@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged.
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>
batch_small.jsonl
-4
View File
@@ -1,4 +0,0 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}
ci_cd/.grype.yaml
+3
View File
@@ -0,0 +1,3 @@
ignore:
- vulnerability: CVE-2019-1010022
reason: no fixed glibc package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
ci_cd/security_scans.sh
+46 -40
View File
@@ -34,47 +34,47 @@ install_ggshield() {
echo "ggshield installed successfully"
}
# Function to run secret detection scans
run_secret_detection() {
echo "Running secret detection scans..."
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
if ! command -v ggshield &> /dev/null; then
install_ggshield
fi
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
if [ -z "$GITGUARDIAN_API_KEY" ]; then
echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
echo "ggshield requires a GitGuardian API key to scan for secrets."
echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
exit 1
fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
echo "Scanning codebase for secrets..."
echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
echo "ggshield will automatically handle rate limits and retry as needed."
echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# Use --recursive for directory scanning and auto-confirm if prompted
# .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# GITGUARDIAN_API_KEY environment variable will be used for authentication
echo y | ggshield secret scan path . --recursive || {
echo ""
echo "=========================================="
echo "ERROR: Secret Detection Failed"
echo "=========================================="
echo "ggshield has detected secrets in the codebase."
echo "Please review discovered secrets above, revoke any actively used secrets"
echo "from underlying systems and make changes to inject secrets dynamically at runtime."
echo ""
echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
echo "=========================================="
echo ""
exit 1
}
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
echo "Secret detection scans completed successfully"
}
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
@@ -101,12 +101,12 @@ run_grype_scans() {
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --fail-on critical
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build --no-cache -t litellm:latest .
grype litellm:latest --fail-on critical
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
@@ -128,6 +128,12 @@ run_grype_scans() {
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
)
# Build JSON array of allowlisted CVE IDs for jq
@@ -208,8 +214,8 @@ main() {
install_trivy
install_grype
echo "Running secret detection scans..."
run_secret_detection
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
+195
View File
@@ -0,0 +1,195 @@
# Claude Code with LiteLLM Quickstart
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
## Video Walkthrough
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Step 1: Setup config.yaml
Create a secure configuration using environment variables:
```yaml
model_list:
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
## Step 2: Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
## Step 3: Verify Setup
Test that your proxy is working correctly:
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
## Step 4: Configure Claude Code
### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
## Step 5: Use Claude Code
Start Claude Code and it will automatically use your configured models:
```bash
# Claude Code will use the models configured in your LiteLLM proxy
claude
# Or specify a model if you have multiple configured
claude --model claude-3-5-sonnet-20241022
claude --model claude-3-5-haiku-20241022
```
## Troubleshooting
Common issues and solutions:
**Claude Code not connecting:**
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` is set correctly
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
- Check LiteLLM logs for detailed error messages
## Using Multiple Models and Providers
Expand your configuration to support multiple providers and models:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Switch between models seamlessly:
```bash
# Use Claude for complex reasoning
claude --model claude-3-5-sonnet-20241022
# Use Haiku for fast responses
claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
```
## Additional Resources
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
cookbook/ai_coding_tool_guides/index.json
+98
View File
@@ -0,0 +1,98 @@
[{
"title": "Claude Code Quickstart",
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
},
{
"title": "Claude Code with MCPs",
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"MCP"
]
},
{
"title": "Claude Code with Non-Anthropic Models",
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"OpenAI",
"Gemini"
]
},
{
"title": "Cursor Quickstart",
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Cursor",
"LiteLLM",
"Quickstart"
]
},
{
"title": "Github Copilot Quickstart",
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Github Copilot",
"LiteLLM",
"Quickstart"
]
},
{
"title": "LiteLLM Gemini CLI Quickstart",
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Gemini CLI",
"Gemini",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenAI Codex CLI Quickstart",
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenAI Codex CLI",
"OpenAI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenWebUI Quickstart",
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenWebUI",
"LiteLLM",
"Quickstart"
]
}]
deploy/Dockerfile.ghcr_base
+2 -1
View File
@@ -8,7 +8,8 @@ WORKDIR /app
COPY config.yaml .
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
# Convert Windows line endings to Unix
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
deploy/charts/litellm-helm/Chart.yaml
+2 -2
View File
@@ -18,13 +18,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.10
version: 1.0.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.50.2
appVersion: v1.80.12
dependencies:
- name: "postgresql"
deploy/charts/litellm-helm/templates/deployment.yaml
+6 -1
View File
@@ -170,7 +170,8 @@ spec:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp
@@ -182,6 +183,10 @@ spec:
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}
deploy/charts/litellm-helm/tests/deployment_tests.yaml
+24 -1
View File
@@ -136,4 +136,27 @@ tests:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
- it: should work with lifecycle hooks
template: deployment.yaml
set:
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- echo "Container stopping"
asserts:
- exists:
path: spec.template.spec.containers[0].lifecycle
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
value: /bin/sh
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"
docker/Dockerfile.alpine
+3 -2
View File
@@ -46,8 +46,9 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
docker/Dockerfile.custom_ui
+3 -2
View File
@@ -32,8 +32,9 @@ RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
WORKDIR /app
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
docker/Dockerfile.database
+11 -7
View File
@@ -27,7 +27,8 @@ RUN python -m pip install build
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -48,7 +49,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
WORKDIR /app
# Copy the current directory contents into the container at /app
@@ -63,20 +64,23 @@ COPY --from=builder /wheels/ /wheels/
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build Admin UI (runtime stage)
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
RUN apk add --no-cache supervisor
docker/Dockerfile.dev
+7 -2
View File
@@ -40,7 +40,8 @@ COPY enterprise/ ./enterprise/
COPY docker/ ./docker/
# Build Admin UI once
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@@ -79,8 +80,12 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
rm -rf /wheels
# Generate prisma client and set permissions
# Convert Windows line endings to Unix for entrypoint scripts
RUN prisma generate && \
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
sed -i 's/\r$//' docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
chmod +x docker/entrypoint.sh && \
chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
docker/Dockerfile.non_root
+19 -14
View File
@@ -40,7 +40,7 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI using the upstream command order while keeping a single RUN layer
RUN mkdir -p /tmp/litellm_ui && \
RUN mkdir -p /var/lib/litellm/ui && \
npm install -g npm@latest && npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
@@ -49,10 +49,10 @@ RUN mkdir -p /tmp/litellm_ui && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
mkdir -p /tmp/litellm_assets && \
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
( cd /tmp/litellm_ui && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
mkdir -p /var/lib/litellm/assets && \
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
( cd /var/lib/litellm/ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
@@ -79,7 +79,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@@ -110,9 +110,11 @@ COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/
# Copy prisma_migration.py for Helm migrations job compatibility
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
COPY --from=builder /app/.cache /app/.cache
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
COPY --from=builder \
@@ -144,9 +146,12 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi
# Permissions, cleanup, and Prisma prep
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
@@ -156,11 +161,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
docs/my-website/docs/anthropic_count_tokens.md
+1
View File
@@ -92,6 +92,7 @@ model_list:
model: vertex_ai/claude-3-5-sonnet-v2@20241022
vertex_project: my-project
vertex_location: us-east5
vertex_count_tokens_location: us-east5 # Optional: Override location for token counting (count_tokens not available on global location)
- model_name: claude-bedrock
litellm_params:
docs/my-website/docs/benchmarks.md
+52
View File
@@ -60,6 +60,58 @@ Each machine deploying LiteLLM had the following specs:
- Database: PostgreSQL
- Redis: Not used
## Infrastructure Recommendations
Recommended specifications based on benchmark results and industry standards for API gateway deployments.
### PostgreSQL
Required for authentication, key management, and usage tracking.
| Workload | CPU | RAM | Storage | Connections |
|----------|-----|-----|---------|-------------|
| 1-2K RPS | 4-8 cores | 16GB | 200GB SSD (3000+ IOPS) | 100-200 |
| 2-5K RPS | 8 cores | 16-32GB | 500GB SSD (5000+ IOPS) | 200-500 |
| 5K+ RPS | 16+ cores | 32-64GB | 1TB+ SSD (10000+ IOPS) | 500+ |
**Configuration:** Set `proxy_batch_write_at: 60` to batch writes and reduce DB load. Total connections = pool limit × instances.
### Redis (Recommended)
Redis was not used in these benchmarks but provides significant production benefits: 60-80% reduced DB load.
| Workload | CPU | RAM |
|----------|-----|-----|
| 1-2K RPS | 2-4 cores | 8GB |
| 2-5K RPS | 4 cores | 16GB |
| 5K+ RPS | 8+ cores | 32GB+ |
**Requirements:** Redis 7.0+, AOF persistence enabled, `allkeys-lru` eviction policy.
**Configuration:**
```yaml
router_settings:
redis_host: os.environ/REDIS_HOST
redis_port: os.environ/REDIS_PORT
redis_password: os.environ/REDIS_PASSWORD
litellm_settings:
cache: True
cache_params:
type: redis
host: os.environ/REDIS_HOST
port: os.environ/REDIS_PORT
password: os.environ/REDIS_PASSWORD
```
:::tip
Use `redis_host`, `redis_port`, and `redis_password` instead of `redis_url` for ~80 RPS better performance.
:::
**Scaling:** DB connections scale linearly with instances. Consider PostgreSQL read replicas beyond 5K RPS.
See [Production Configuration](./proxy/prod) for detailed best practices.
## Locust Settings
- 1000 Users
docs/my-website/docs/completion/input.md
+41 -1
View File
@@ -142,7 +142,47 @@ def completion(
- `tool_call_id`: *str (optional)* - Tool call that this message is responding to.
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392)
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664)
#### Content Types
`content` can be a string (text only) or a list of content blocks (multimodal):
| Type | Description | Docs |
|------|-------------|------|
| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) |
| `image_url` | Images | [Vision](./vision.md) |
| `input_audio` | Audio input | [Audio](./audio.md) |
| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) |
| `file` | Files | [Document Understanding](./document_understanding.md) |
| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) |
**Examples:**
```python
# Text
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}]
# Image
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}]
# Audio
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "<base64>", "format": "wav"}}]}]
# Video
messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}]
# File
messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}]
# Document
messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": "<base64>"}}]}]
# Combining multiple types (multimodal)
messages=[{"role": "user", "content": [
{"type": "text", "text": "Generate a product description based on this image"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]}]
```
## Optional Fields
docs/my-website/docs/container_files.md
+81
View File
@@ -21,6 +21,7 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/v1/containers/{container_id}/files` | POST | Upload file to container |
| `/v1/containers/{container_id}/files` | GET | List files in container |
| `/v1/containers/{container_id}/files/{file_id}` | GET | Get file metadata |
| `/v1/containers/{container_id}/files/{file_id}/content` | GET | Download file content |
@@ -28,6 +29,45 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
## LiteLLM Python SDK
### Upload Container File
Upload files directly to a container session. This is useful when `/chat/completions` or `/responses` sends files to the container but the input file type is limited to PDF. This endpoint lets you work with other file types like CSV, Excel, Python scripts, etc.
```python showLineNumbers title="upload_container_file.py"
from litellm import upload_container_file
# Upload a CSV file
file = upload_container_file(
container_id="cntr_123...",
file=("data.csv", open("data.csv", "rb").read(), "text/csv"),
custom_llm_provider="openai"
)
print(f"Uploaded: {file.id}")
print(f"Path: {file.path}")
```
**Async:**
```python showLineNumbers title="aupload_container_file.py"
from litellm import aupload_container_file
file = await aupload_container_file(
container_id="cntr_123...",
file=("script.py", b"print('hello world')", "text/x-python"),
custom_llm_provider="openai"
)
```
**Supported file formats:**
- CSV (`.csv`)
- Excel (`.xlsx`)
- Python scripts (`.py`)
- JSON (`.json`)
- Markdown (`.md`)
- Text files (`.txt`)
- And more...
### List Container Files
```python showLineNumbers title="list_container_files.py"
@@ -103,6 +143,40 @@ print(f"Deleted: {result.deleted}")
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
### Upload File
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="upload_file.py"
from openai import OpenAI
client = OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
file = client.containers.files.create(
container_id="cntr_123...",
file=open("data.csv", "rb")
)
print(f"Uploaded: {file.id}")
print(f"Path: {file.path}")
```
</TabItem>
<TabItem value="curl" label="curl">
```bash showLineNumbers title="upload_file.sh"
curl "http://localhost:4000/v1/containers/cntr_123.../files" \
-H "Authorization: Bearer sk-1234" \
-F file="@data.csv"
```
</TabItem>
</Tabs>
### List Files
<Tabs>
@@ -236,6 +310,13 @@ curl -X DELETE "http://localhost:4000/v1/containers/cntr_123.../files/cfile_456.
## Parameters
### Upload File
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `container_id` | string | Yes | Container ID |
| `file` | FileTypes | Yes | File to upload. Can be a tuple of (filename, content, content_type), file-like object, or bytes |
### List Files
| Parameter | Type | Required | Description |
docs/my-website/docs/data_retention.md
+1 -1
View File
@@ -10,7 +10,7 @@ This policy outlines the requirements and controls/procedures LiteLLM Cloud has
For Customers
1. Active Accounts
- Customer data is retained for as long as the customer’s account is in active status. This includes data such as prompts, generated content, logs, and usage metrics.
- Customer data is retained for as long as the customer’s account is in active status. This includes data such as prompts, generated content, logs, and usage metrics. By default, we do not store the message / response content of your API requests or responses. Cloud users need to explicitly opt in to store the message / response content of your API requests or responses.
2. Voluntary Account Closure
docs/my-website/docs/image_generation.md
+23 -1
View File
@@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
## Quick Start
@@ -238,6 +238,27 @@ print(response)
See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
## OpenRouter Image Generation Models
Use this for image generation models available through OpenRouter (e.g., Google Gemini image generation models)
#### Usage
```python showLineNumbers
from litellm import image_generation
import os
os.environ['OPENROUTER_API_KEY'] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
size="1024x1024",
quality="high",
)
print(response)
```
## OpenAI Compatible Image Generation Models
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
@@ -301,5 +322,6 @@ print(f"response: {response}")
| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
| OpenRouter | [OpenRouter Image Generation →](./providers/openrouter#image-generation) |
| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) |
docs/my-website/docs/interactions.md
+56 -1
View File
@@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) |
| Supported LLM providers | **All LiteLLM supported CHAT COMPLETION providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
## **LiteLLM Python SDK Usage**
@@ -207,8 +207,63 @@ for chunk in client.interactions.create_stream(
}
```
## **Calling non-Interactions API endpoints (`/interactions` to `/responses` Bridge)**
LiteLLM allows you to call non-Interactions API models via a bridge to LiteLLM's `/responses` endpoint. This is useful for calling OpenAI, Anthropic, and other providers that don't natively support the Interactions API.
#### Python SDK Usage
```python showLineNumbers title="SDK Usage"
import litellm
import os
# Set API key
os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
# Non-streaming interaction
response = litellm.interactions.create(
model="gpt-4o",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
```
#### LiteLLM Proxy Usage
**Setup Config:**
```yaml showLineNumbers title="Example Configuration"
model_list:
- model_name: openai-model
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
**Start Proxy:**
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
**Make Request:**
```bash showLineNumbers title="non-Interactions API Model Request"
curl http://localhost:4000/v1beta/interactions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "openai-model",
"input": "Tell me a short joke about programming."
}'
```
## **Supported Providers**
| Provider | Link to Usage |
|----------|---------------|
| Google AI Studio | [Usage](#quick-start) |
| All other LiteLLM providers | [Bridge Usage](#calling-non-interactions-api-endpoints-interactions-to-responses-bridge) |
docs/my-website/docs/mcp.md
+62 -3
View File
@@ -17,7 +17,7 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
## Overview
| Feature | Description |
|---------|-------------|
| MCP Operations | • List Tools<br/>• Call Tools |
| MCP Operations | • List Tools<br/>• Call Tools <br/>• Prompts <br/>• Resources |
| Supported MCP Transports | • Streamable HTTP<br/>• SSE<br/>• Standard Input/Output (stdio) |
| LiteLLM Permission Management | • By Key<br/>• By Team<br/>• By Organization |
@@ -60,6 +60,8 @@ model_list:
If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
For diagnosing connectivity problems after setup, see the [MCP Troubleshooting Guide](./mcp_troubleshoot.md).
<Tabs>
<TabItem value="ui" label="LiteLLM UI">
@@ -110,6 +112,22 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t
<br/>
<br/>
### OAuth Configuration & Overrides
LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.ietf.org/doc/html/rfc8414) by default. When you create an MCP server in the UI and set `Authentication: OAuth`, LiteLLM will locate the provider metadata, dynamically register a client, and perform PKCE-based authorization without you providing any additional details.
**Customize the OAuth flow when needed:**
<Image
img={require('../img/mcp_oauth.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
- **Provide explicit client credentials** – If the MCP provider does not offer dynamic client registration or you prefer to manage the client yourself, fill in `client_id`, `client_secret`, and the desired `scopes`.
- **Override discovery URLs** – In some environments, LiteLLM might not be able to reach the provider's metadata endpoints. Use the optional `authorization_url`, `token_url`, and `registration_url` fields to point LiteLLM directly to the correct endpoints.
<br/>
### Static Headers
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
@@ -182,6 +200,7 @@ mcp_servers:
- `http` - Streamable HTTP transport
- `stdio` - Standard Input/Output transport
- **Command**: The command to execute for stdio transport (required for stdio)
- **allow_all_keys**: Set to `true` to make the server available to every LiteLLM API key, even if the key/team doesn't list the server in its MCP permissions.
- **Args**: Array of arguments to pass to the command (optional for stdio)
- **Env**: Environment variables to set for the stdio process (optional for stdio)
- **Description**: Optional description for the server
@@ -309,6 +328,7 @@ litellm_settings:
</TabItem>
</Tabs>
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
@@ -485,7 +505,7 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
This configuration is currently available on the config.yaml, with UI support coming soon.
You can configure this either in `config.yaml` or directly from the LiteLLM UI (MCP Servers → Authentication → OAuth).
```yaml
mcp_servers:
@@ -746,8 +766,33 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \
3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server
4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers
---
### Passing Request Headers to STDIO env Vars
If your stdio MCP server needs per-request credentials, you can map HTTP headers from the client request directly into the environment for the launched stdio process. Reference the header name in the env value using the `${X-HEADER_NAME}` syntax. LiteLLM will read that header from the incoming request and set the env var before starting the command.
```json title="Forward X-GITHUB_PERSONAL_ACCESS_TOKEN header to stdio env" showLineNumbers
{
"mcpServers": {
"github": {
"command": "docker",
"args": [
"run",
"-i",
"--rm",
"-e",
"GITHUB_PERSONAL_ACCESS_TOKEN",
"ghcr.io/github/github-mcp-server"
],
"env": {
"GITHUB_PERSONAL_ACCESS_TOKEN": "${X-GITHUB_PERSONAL_ACCESS_TOKEN}"
}
}
}
}
```
In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
## Using your MCP with client side credentials
@@ -1431,3 +1476,17 @@ async with stdio_client(server_params) as (read, write):
</TabItem>
</Tabs>
## FAQ
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**
The UI keeps only transient state in `sessionStorage` so the OAuth redirect flow can finish; the token is not persisted in the server or database.
**Q: I'm seeing MCP connection errors—what should I check?**
Walk through the [MCP Troubleshooting Guide](./mcp_troubleshoot.md) for step-by-step isolation (Client → LiteLLM vs. LiteLLM → MCP), log examples, and verification methods like MCP Inspector and `curl`.
docs/my-website/docs/mcp_control.md
+71
View File
@@ -13,6 +13,7 @@ LiteLLM provides fine-grained permission management for MCP servers, allowing yo
- **Restrict MCP access by entity**: Control which keys, teams, or organizations can access specific MCP servers
- **Tool-level filtering**: Automatically filter available tools based on entity permissions
- **Centralized control**: Manage all MCP permissions from the LiteLLM Admin UI or API
- **One-click public MCPs**: Mark specific servers as available to every LiteLLM API key when you don't need per-key restrictions
This ensures that only authorized entities can discover and use MCP tools, providing an additional security layer for your MCP infrastructure.
@@ -95,6 +96,48 @@ mcp_servers:
- If you specify both `allowed_tools` and `disallowed_tools`, the allowed list takes priority
- Tool names are case-sensitive
## Public MCP Servers (allow_all_keys)
Some MCP servers are meant to be shared broadly—think internal knowledge bases, calendar integrations, or other low-risk utilities where every team should be able to connect without requesting access. Instead of adding those servers to every key, team, or organization, enable the new `allow_all_keys` toggle.
<Tabs>
<TabItem value="ui" label="UI">
1. Open **MCP Servers → Add / Edit** in the Admin UI.
2. Expand **Permission Management / Access Control**.
3. Toggle **Allow All LiteLLM Keys** on.
<Image
img={require('../img/mcp_allow_all_ui.png')}
style={{width: '80%', display: 'block', margin: '1rem auto'}}
alt="MCP server configuration in Admin UI"
/>
The toggle makes the server “public” without touching existing access groups.
</TabItem>
<TabItem value="config" label="config.yaml">
Set `allow_all_keys: true` to mark the server as public:
```yaml title="Make an MCP server public" showLineNumbers
mcp_servers:
deepwiki:
url: https://mcp.deepwiki.com/mcp
allow_all_keys: true
```
</TabItem>
</Tabs>
### When to use it
- You have shared MCP utilities where fine-grained ACLs would only add busywork.
- You want a “default enabled” experience for internal users, while still being able to layer tool-level restrictions.
- You’re onboarding new teams and want the safest MCPs available out of the box.
Once enabled, LiteLLM automatically includes the server for every key during tool discovery/calls—no extra virtual-key or team configuration is required.
---
## Allow/Disallow MCP Tool Parameters
@@ -591,3 +634,31 @@ Control which tools different teams can access from the same MCP server. For exa
This video shows how to set allowed tools for a Key, Team, or Organization.
<iframe width="840" height="500" src="https://www.loom.com/embed/7464d444c3324078892367272fe50745" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Dashboard View Modes
Proxy admins can also control what non-admins see inside the MCP dashboard via `general_settings.user_mcp_management_mode`:
- `restricted` *(default)* – users only see servers that their team explicitly has access to.
- `view_all` – every dashboard user can see the full MCP server list.
```yaml title="Config example"
general_settings:
user_mcp_management_mode: view_all
```
This is useful when you want discoverability for MCP offerings without granting additional execution privileges.
## Publish MCP Registry
If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry).
1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy.
2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`.
3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL.
:::note Permissions still apply
The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions.
:::
docs/my-website/docs/mcp_guardrail.md
+1
View File
@@ -85,4 +85,5 @@ MCP guardrails work with all LiteLLM-supported guardrail providers:
- **Bedrock**: AWS Bedrock guardrails
- **Lakera**: Content moderation
- **Aporia**: Custom guardrails
- **Noma**: Noma Security
- **Custom**: Your own guardrail implementations
docs/my-website/docs/mcp_troubleshoot.md
+99
View File
@@ -0,0 +1,99 @@
import Image from '@theme/IdealImage';
# MCP Troubleshooting Guide
When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Proxy → MCP Server`, while OAuth-enabled setups add an authorization server for metadata discovery.
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
## Locate the Error Source
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
<Image
img={require('../img/mcp_tool_testing_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
**Actions**
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
### Client Traffic Issues (Client → LiteLLM)
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
#### MCP Protocol Sessions
Clients such as IDEs or agent runtimes speak the MCP protocol directly with LiteLLM.
**Actions**
- Inspect LiteLLM access logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to verify the client request reached the proxy and which MCP server it targeted.
- Review LiteLLM error logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) for TLS, authentication, or routing errors that block the request before the MCP call starts.
- Use the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to confirm the MCP server is reachable outside of the failing client.
#### Responses/Completions with Embedded MCP Calls
During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls mid-request. An error could occur before the MCP call begins or after the MCP responds.
**Actions**
- Check LiteLLM request logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to see whether an MCP attempt was recorded; if not, the problem lies in `Client → LiteLLM`.
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
<Image
img={require('../img/mcp_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
### OAuth Metadata Discovery
LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://modelcontextprotocol.info/specification/draft/basic/authorization/#23-server-metadata-discovery)). When OAuth is enabled, confirm the authorization server exposes the metadata URL and that LiteLLM can fetch it.
**Actions**
- Use `curl <metadata_url>` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
## Verify Connectivity
Run lightweight validations before impacting production traffic.
### MCP Inspector
Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Client → MCP` communications in one place; it makes isolating the failing hop straightforward.
1. Execute `npx @modelcontextprotocol/inspector` on your workstation.
2. Configure and connect:
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
- **Custom Headers:** e.g., `Authorization: Bearer <LiteLLM API Key>`.
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
### `curl` Smoke Test
`curl` is ideal on servers where installing the Inspector is impractical. It replicates the MCP tool call LiteLLM would make—swap in the domain of the system under test (LiteLLM or the MCP server).
```bash
curl -X POST https://your-target-domain.example.com/mcp \
-H "Content-Type: application/json" \
-H "Accept: application/json, text/event-stream" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
Add `-H "Authorization: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
## Review Logs
Well-scoped logs make it clear whether LiteLLM reached the MCP server and what happened next.
### Access Log Example (successful MCP call)
```text
INFO: 127.0.0.1:57230 - "POST /everything/mcp HTTP/1.1" 200 OK
```
### Error Log Example (failed MCP call)
```text
07:22:00 - LiteLLM:ERROR: client.py:224 - MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception), Server: http://localhost:3001/mcp, Transport: MCPTransport.http
httpcore.ConnectError: All connection attempts failed
ERROR:LiteLLM:MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception)...
httpx.ConnectError: All connection attempts failed
```
docs/my-website/docs/observability/arize_integration.md
+1
View File
@@ -68,6 +68,7 @@ environment_variables:
ARIZE_API_KEY: "141a****"
ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint
ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc)
ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name
```
2. Start the proxy
docs/my-website/docs/observability/cloudzero.md
+46
View File
@@ -65,6 +65,52 @@ Start your LiteLLM proxy with the configuration:
litellm --config /path/to/config.yaml
```
## Setup on UI
1\. Click "Settings"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/5ac36280-c688-41a3-8d0e-23e19c6a470b/ascreenshot.jpeg?tl_px=0,332&br_px=1308,1064&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=119,444)
2\. Click "Logging & Alerts"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/13f76b09-e0c4-4738-ba05-2d5111c6ad3e/ascreenshot.jpeg?tl_px=0,332&br_px=1308,1064&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=58,507)
3\. Click "CloudZero Cost Tracking"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/f96cc1e5-7bc0-4d7c-9aeb-5cbbec549b12/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=389,56)
4\. Click "Add CloudZero Integration"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/04fbc748-0e6f-43bb-8a57-dd2e83dbfcb5/ascreenshot.jpeg?tl_px=0,90&br_px=1308,821&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=616,277)
5\. Enter your CloudZero API Key.
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/080e82f1-f94f-4ed7-8014-e495380336f3/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=506,129)
6\. Enter your CloudZero Connection ID.
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/af417aa2-67a8-4dee-a014-84b1892dc07e/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=488,213)
7\. Click "Create"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/647e672f-9a4a-4754-a7b0-abf1397abad4/ascreenshot.jpeg?tl_px=0,88&br_px=1308,819&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=711,277)
8\. Test your payload with "Run Dry Run Simulation"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/7447cbe0-3450-4be5-bdc4-37fb8280aa58/ascreenshot.jpeg?tl_px=0,125&br_px=1308,856&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=334,277)
10\. Click "Export Data Now" to export to CLoudZero
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/7be9bd48-6e27-4c68-bc75-946f3ab593d9/ascreenshot.jpeg?tl_px=0,130&br_px=1308,861&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=518,277)
## Testing Your Setup
### Dry Run Export
docs/my-website/docs/observability/focus.md
+93
View File
@@ -0,0 +1,93 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Focus Export (Experimental)
:::caution Experimental feature
Focus Format export is under active development and currently considered experimental.
Interfaces, schema mappings, and configuration options may change as we iterate based on user feedback.
Please treat this integration as a preview and report any issues or suggestions to help us stabilize and improve the workflow.
:::
LiteLLM can emit usage data in the [FinOps FOCUS format](https://focus.finops.org/focus-specification/v1-2/) and push artifacts (for example Parquet files) to destinations such as Amazon S3. This enables downstream cost-analysis tooling to ingest a standardised dataset directly from LiteLLM.
LiteLLM currently conforms to the FinOps FOCUS v1.2 specification when emitting this dataset.
## Overview
| Property | Details |
|----------|---------|
| Destination | Export LiteLLM usage data in FOCUS format to managed storage (currently S3) |
| Callback name | `focus` |
| Supported operations | Automatic scheduled export |
| Data format | FOCUS Normalised Dataset (Parquet) |
## Environment Variables
### Common settings
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_PROVIDER` | No | Destination provider (defaults to `s3`). |
| `FOCUS_FORMAT` | No | Output format (currently only `parquet`). |
| `FOCUS_FREQUENCY` | No | Export cadence. Prefer `hourly` or `daily` for production; `interval` is intended for short test loops. Defaults to `hourly`. |
| `FOCUS_CRON_OFFSET` | No | Minute offset used for hourly/daily cron triggers. Defaults to `5`. |
| `FOCUS_INTERVAL_SECONDS` | No | Interval (seconds) when `FOCUS_FREQUENCY="interval"`. |
| `FOCUS_PREFIX` | No | Object key prefix/folder. Defaults to `focus_exports`. |
### S3 destination
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_S3_BUCKET_NAME` | Yes | Destination bucket for exported files. |
| `FOCUS_S3_REGION_NAME` | No | AWS region for the bucket. |
| `FOCUS_S3_ENDPOINT_URL` | No | Custom endpoint (useful for S3-compatible storage). |
| `FOCUS_S3_ACCESS_KEY` | Yes | AWS access key for uploads. |
| `FOCUS_S3_SECRET_KEY` | Yes | AWS secret key for uploads. |
| `FOCUS_S3_SESSION_TOKEN` | No | AWS session token if using temporary credentials. |
## Setup via Config
### Configure environment variables
```bash
export FOCUS_PROVIDER="s3"
export FOCUS_PREFIX="focus_exports"
# S3 example
export FOCUS_S3_BUCKET_NAME="my-litellm-focus-bucket"
export FOCUS_S3_REGION_NAME="us-east-1"
export FOCUS_S3_ACCESS_KEY="AKIA..."
export FOCUS_S3_SECRET_KEY="..."
```
### Update LiteLLM config
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-your-key
litellm_settings:
callbacks: ["focus"]
```
### Start the proxy
```bash
litellm --config /path/to/config.yaml
```
During boot LiteLLM registers the Focus logger and a background job that runs according to the configured frequency.
## Planned Enhancements
- Add "Setup on UI" flow alongside the current configuration-based setup.
- Add GCS / Azure Blob to the Destination options.
- Support CSV output alongside Parquet.
## Related Links
- [Focus](https://focus.finops.org/)
docs/my-website/docs/observability/generic_api.md
+59
View File
@@ -47,6 +47,7 @@ callback_settings:
| `endpoint` | string | Yes | HTTP endpoint to send logs to |
| `headers` | dict | No | Custom headers for the request |
| `event_types` | list | No | Filter events: `llm_api_success`, `llm_api_failure`. Defaults to all events. |
| `log_format` | string | No | Output format: `json_array` (default), `ndjson`, or `single`. Controls how logs are batched and sent. |
## Pre-configured Callbacks
@@ -107,4 +108,62 @@ callback_settings:
flush_interval: 60 # seconds, default: 60
```
## Log Format Options
Control how logs are formatted and sent to your endpoint.
### JSON Array (Default)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: json_array # default if not specified
```
Sends all logs in a batch as a single JSON array `[{log1}, {log2}, ...]`. This is the default behavior and maintains backward compatibility.
**When to use**: Most HTTP endpoints expecting batched JSON data.
### NDJSON (Newline-Delimited JSON)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: ndjson
```
Sends logs as newline-delimited JSON (one record per line):
```
{log1}
{log2}
{log3}
```
**When to use**: Log aggregation services like Sumo Logic, Splunk, or Datadog that support field extraction on individual records.
**Benefits**:
- Each log is ingested as a separate message
- Field Extraction Rules work at ingest time
- Better parsing and querying performance
### Single
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: single
```
Sends each log as an individual HTTP request in parallel when the batch is flushed.
**When to use**: Endpoints that expect individual records, or when you need maximum compatibility.
**Note**: This mode sends N HTTP requests per batch (more overhead). Consider using `ndjson` instead if your endpoint supports it.
docs/my-website/docs/observability/levo_integration.md
+162
View File
@@ -0,0 +1,162 @@
---
sidebar_label: Levo AI
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Levo AI
<div className="levo-logo-container" style={{ marginTop: '0.5rem', marginBottom: '1rem' }}>
<div className="levo-logo-light">
<Image img={require('../../img/levo_logo.png')} />
</div>
<div className="levo-logo-dark">
<Image img={require('../../img/levo_logo_dark.png')} />
</div>
</div>
[Levo](https://levo.ai/) is an AI observability and compliance platform that provides comprehensive monitoring, analysis, and compliance tracking for LLM applications.
## Quick Start
Send all your LLM requests and responses to Levo for monitoring and analysis using LiteLLM's built-in Levo integration.
### What You'll Get
- **Complete visibility** into all LLM API calls across all providers
- **Request and response data** including prompts, completions, and metadata
- **Usage and cost tracking** with token counts and cost breakdowns
- **Error monitoring** and performance metrics
- **Compliance tracking** for audit and governance
### Setup Steps
**1. Install OpenTelemetry dependencies:**
```bash
pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc
```
**2. Enable Levo callback in your LiteLLM config:**
Add to your `litellm_config.yaml`:
```yaml
litellm_settings:
callbacks: ["levo"]
```
**3. Configure environment variables:**
[Contact Levo support](mailto:support@levo.ai) to get your collector endpoint URL, API key, organization ID, and workspace ID.
Set these required environment variables:
```bash
export LEVOAI_API_KEY="<your-levo-api-key>"
export LEVOAI_ORG_ID="<your-levo-org-id>"
export LEVOAI_WORKSPACE_ID="<your-workspace-id>"
export LEVOAI_COLLECTOR_URL="<your-levo-collector-url>"
```
**Note:** The collector URL should be the full endpoint URL provided by Levo support. It will be used exactly as provided.
**4. Start LiteLLM:**
```bash
litellm --config config.yaml
```
**5. Make requests - they'll automatically be sent to Levo!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "Hello, this is a test message"
}
]
}'
```
## What Data is Captured
| Feature | Details |
|---------|---------|
| **What is logged** | OpenTelemetry Trace Data (OTLP format) |
| **Events** | Success + Failure |
| **Format** | OTLP (OpenTelemetry Protocol) |
| **Headers** | Automatically includes `Authorization: Bearer {LEVOAI_API_KEY}`, `x-levo-organization-id`, and `x-levo-workspace-id` |
## Configuration Reference
### Required Environment Variables
| Variable | Description | Example |
|----------|-------------|---------|
| `LEVOAI_API_KEY` | Your Levo API key | `levo_abc123...` |
| `LEVOAI_ORG_ID` | Your Levo organization ID | `org-123456` |
| `LEVOAI_WORKSPACE_ID` | Your Levo workspace ID | `workspace-789` |
| `LEVOAI_COLLECTOR_URL` | Full collector endpoint URL from Levo support | `https://collector.levo.ai/v1/traces` |
### Optional Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `LEVOAI_ENV_NAME` | Environment name for tagging traces | `None` |
**Note:** The collector URL is used exactly as provided by Levo support. No path manipulation is performed.
## Troubleshooting
### Not seeing traces in Levo?
1. **Verify Levo callback is enabled**: Check LiteLLM startup logs for `initializing callbacks=['levo']`
2. **Check required environment variables**: Ensure all required variables are set:
```bash
echo $LEVOAI_API_KEY
echo $LEVOAI_ORG_ID
echo $LEVOAI_WORKSPACE_ID
echo $LEVOAI_COLLECTOR_URL
```
3. **Verify collector connectivity**: Test if your collector is reachable:
```bash
curl <your-collector-url>/health
```
4. **Check for initialization errors**: Look for errors in LiteLLM startup logs. Common issues:
- Missing OpenTelemetry packages: Install with `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
- Missing required environment variables: All four required variables must be set
- Invalid collector URL: Ensure the URL is correct and reachable
5. **Enable debug logging**:
```bash
export LITELLM_LOG="DEBUG"
```
6. **Wait for async export**: OTLP sends traces asynchronously. Wait 10-15 seconds after making requests before checking Levo.
### Common Errors
**Error: "LEVOAI_COLLECTOR_URL environment variable is required"**
- Solution: Set the `LEVOAI_COLLECTOR_URL` environment variable with your collector endpoint URL from Levo support.
**Error: "No module named 'opentelemetry'"**
- Solution: Install OpenTelemetry packages: `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
## Additional Resources
- [Levo Documentation](https://docs.levo.ai)
- [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/)
## Need Help?
For issues or questions about the Levo integration with LiteLLM, please [contact Levo support](mailto:support@levo.ai) or open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm/issues).
docs/my-website/docs/observability/logfire_integration.md
+4
View File
@@ -40,6 +40,10 @@ import os
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_TOKEN"] = ""
# Optionally customize the base url
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_BASE_URL"] = ""
# LLM API Keys
os.environ['OPENAI_API_KEY']=""
docs/my-website/docs/observability/opentelemetry_integration.md
+4 -2
View File
@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# OpenTelemetry - Tracing LLMs with any observability tool
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop and others.
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop, Levo AI and others.
<Image img={require('../../img/traceloop_dash.png')} />
@@ -12,7 +12,9 @@ OpenTelemetry is a CNCF standard for observability. It connects to any observabi
From v1.81.0, the request/response will be set as attributes on the parent "Received Proxy Server Request" span by default. This allows you to see the request/response in the parent span in your observability tool.
To use the older behavior with nested "litellm_request" spans, set the following environment variable:
**Note:** When making multiple LLM calls within an external OTEL span context, the last call's attributes will overwrite previous calls' attributes on the parent span.
To use the older behavior with nested "litellm_request" spans (which creates separate spans for each call), set the following environment variable:
```shell
USE_OTEL_LITELLM_REQUEST_SPAN=true
docs/my-website/docs/observability/qualifire_integration.md
+122
View File
@@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
# Qualifire - LLM Evaluation, Guardrails & Observability
[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications.
**Key Features:**
- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities
- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches
- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents
- **Prompt Management** - Centralized prompt management with versioning and no-code studio
:::tip
Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more.
:::
## Pre-Requisites
1. Create an account on [Qualifire](https://app.qualifire.ai/)
2. Get your API key and webhook URL from the Qualifire dashboard
```bash
pip install litellm
```
## Quick Start
Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire.
```python
litellm.callbacks = ["qualifire_eval"]
```
```python
import litellm
import os
# Set Qualifire credentials
os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key"
os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = "your-openai-api-key"
# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire
litellm.callbacks = ["qualifire_eval"]
# OpenAI call
response = litellm.completion(
model="gpt-5",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
]
)
```
## Using with LiteLLM Proxy
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["qualifire_eval"]
general_settings:
master_key: "sk-1234"
environment_variables:
QUALIFIRE_API_KEY: "your-qualifire-api-key"
QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations"
```
2. Start the proxy
```bash
litellm --config config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
```
## Environment Variables
| Variable | Description |
| ----------------------- | ------------------------------------------------------ |
| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication |
| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard |
## What Gets Logged?
The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call.
This includes:
- Request messages and parameters
- Response content and metadata
- Token usage statistics
- Latency metrics
- Model information
- Cost data
Once data is in Qualifire, you can:
- Run evaluations to detect hallucinations, toxicity, and policy violations
- Set up guardrails to block or modify responses in real-time
- View traces across your entire AI pipeline
- Track performance and quality metrics over time
docs/my-website/docs/observability/signoz.md
+394
View File
@@ -0,0 +1,394 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# SigNoz LiteLLM Integration
For more details on setting up observability for LiteLLM, check out the [SigNoz LiteLLM observability docs](https://signoz.io/docs/litellm-observability/).
## Overview
This guide walks you through setting up observability and monitoring for LiteLLM SDK and Proxy Server using [OpenTelemetry](https://opentelemetry.io/) and exporting logs, traces, and metrics to SigNoz. With this integration, you can observe various models performance, capture request/response details, and track system-level metrics in SigNoz, giving you real-time visibility into latency, error rates, and usage trends for your LiteLLM applications.
Instrumenting LiteLLM in your AI applications with telemetry ensures full observability across your AI workflows, making it easier to debug issues, optimize performance, and understand user interactions. By leveraging SigNoz, you can analyze correlated traces, logs, and metrics in unified dashboards, configure alerts, and gain actionable insights to continuously improve reliability, responsiveness, and user experience.
## Prerequisites
- A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key
- Internet access to send telemetry data to SigNoz Cloud
- [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration
- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies
## Monitoring LiteLLM
LiteLLM can be monitored in two ways: using the **LiteLLM SDK** (directly embedded in your Python application code for programmatic LLM calls) or the **LiteLLM Proxy Server** (a standalone server that acts as a centralized gateway for managing and routing LLM requests across your infrastructure).
<Tabs>
<TabItem value="LiteLLM SDK" label="LiteLLM SDK" default>
For more detailed info on instrumenting your LiteLLM SDK applications click [here](https://docs.litellm.ai/docs/observability/opentelemetry_integration).
<Tabs>
<TabItem value="No Code" label="No Code(Recommended)" default>
No-code auto-instrumentation is recommended for quick setup with minimal code changes. It's ideal when you want to get observability up and running without modifying your application code and are leveraging standard instrumentor libraries.
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install \
opentelemetry-api \
opentelemetry-distro \
opentelemetry-exporter-otlp \
httpx \
opentelemetry-instrumentation-httpx \
litellm
```
**Step 2:** Add Automatic Instrumentation
```bash
opentelemetry-bootstrap --action=install
```
**Step 3:** Instrument your LiteLLM SDK application
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
```python
from litellm import litellm
litellm.callbacks = ["otel"]
```
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
**Step 4:** Run an example
```python
from litellm import completion, litellm
litellm.callbacks = ["otel"]
response = completion(
model="openai/gpt-4o",
messages=[{ "content": "What is SigNoz","role": "user"}]
)
print(response)
```
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
**Step 5:** Run your application with auto-instrumentation
```bash
OTEL_RESOURCE_ATTRIBUTES="service.name=<service_name>" \
OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443" \
OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>" \
OTEL_EXPORTER_OTLP_PROTOCOL=grpc \
OTEL_TRACES_EXPORTER=otlp \
OTEL_METRICS_EXPORTER=otlp \
OTEL_LOGS_EXPORTER=otlp \
OTEL_PYTHON_LOG_CORRELATION=true \
OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \
OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \
opentelemetry-instrument <your_run_command>
```
> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation.
- **`<service_name>`** is the name of your service
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
- Replace `<your_run_command>` with the actual command you would use to run your application. For example: `python main.py`
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
</TabItem>
<TabItem value="Code" label="Code" default>
Code-based instrumentation gives you fine-grained control over your telemetry configuration. Use this approach when you need to customize resource attributes, sampling strategies, or integrate with existing observability infrastructure.
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install \
opentelemetry-api \
opentelemetry-sdk \
opentelemetry-exporter-otlp \
opentelemetry-instrumentation-httpx \
opentelemetry-instrumentation-system-metrics \
litellm
```
**Step 2:** Import the necessary modules in your Python application
**Traces:**
```python
from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
```
**Logs:**
```python
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry._logs import set_logger_provider
import logging
```
**Metrics:**
```python
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry import metrics
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
```
**Step 3:** Set up the OpenTelemetry Tracer Provider to send traces directly to SigNoz Cloud
```python
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry import trace
import os
resource = Resource.create({"service.name": "<service_name>"})
provider = TracerProvider(resource=resource)
span_exporter = OTLPSpanExporter(
endpoint= os.getenv("OTEL_EXPORTER_TRACES_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
processor = BatchSpanProcessor(span_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_TRACES_ENDPOINT`** → SigNoz Cloud trace endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/traces`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 4**: Setup Logs
```python
import logging
from opentelemetry.sdk.resources import Resource
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
import os
resource = Resource.create({"service.name": "<service_name>"})
logger_provider = LoggerProvider(resource=resource)
set_logger_provider(logger_provider)
otlp_log_exporter = OTLPLogExporter(
endpoint= os.getenv("OTEL_EXPORTER_LOGS_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(otlp_log_exporter)
)
# Attach OTel logging handler to root logger
handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
logging.basicConfig(level=logging.INFO, handlers=[handler])
logger = logging.getLogger(__name__)
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_LOGS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/logs`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 5**: Setup Metrics
```python
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry import metrics
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
import os
resource = Resource.create({"service.name": "<service-name>"})
metric_exporter = OTLPMetricExporter(
endpoint= os.getenv("OTEL_EXPORTER_METRICS_ENDPOINT"),
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
)
reader = PeriodicExportingMetricReader(metric_exporter)
metric_provider = MeterProvider(metric_readers=[reader], resource=resource)
metrics.set_meter_provider(metric_provider)
meter = metrics.get_meter(__name__)
# turn on out-of-the-box metrics
SystemMetricsInstrumentor().instrument()
HTTPXClientInstrumentor().instrument()
```
- **`<service_name>`** is the name of your service
- **`OTEL_EXPORTER_METRICS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/metrics`
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
> 📌 Note: SystemMetricsInstrumentor provides system metrics (CPU, memory, etc.), and HTTPXClientInstrumentor provides outbound HTTP request metrics such as request duration. If you want to add custom metrics to your LiteLLM application, see [Python Custom Metrics](https://signoz.io/opentelemetry/python-custom-metrics/).
**Step 6:** Instrument your LiteLLM application
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
```python
from litellm import litellm
litellm.callbacks = ["otel"]
```
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
**Step 7:** Run an example
```python
from litellm import completion, litellm
litellm.callbacks = ["otel"]
response = completion(
model="openai/gpt-4o",
messages=[{ "content": "What is SigNoz","role": "user"}]
)
print(response)
```
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
</TabItem>
</Tabs>
## View Traces, Logs, and Metrics in SigNoz
Your LiteLLM commands should now automatically emit traces, logs, and metrics.
You should be able to view traces in Signoz Cloud under the traces tab:
![LiteLLM SDK Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-traces.webp)
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
![LiteLLM SDK Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-traces.webp)
You should be able to view logs in Signoz Cloud under the logs tab. You can also view logs by clicking on the “Related Logs” button in the trace view to see correlated logs:
![LiteLLM SDK Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-logs.webp)
When you click on any of these logs in SigNoz, you'll see a detailed view of the log, including attributes:
![LiteLLM SDK Detailed Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-logs.webp)
You should be able to see LiteLLM related metrics in Signoz Cloud under the metrics tab:
![LiteLLM SDK Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-metrics.webp)
When you click on any of these metrics in SigNoz, you'll see a detailed view of the metric, including attributes:
![LiteLLM Detailed Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-metrics.webp)
## Dashboard
You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-sdk-dashboard/) which provides specialized visualizations for monitoring your LiteLLM usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
![LiteLLM SDK Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-sdk-dashboard.webp)
</TabItem>
<TabItem value="LiteLLM Proxy Server" label="LiteLLM Proxy Server" default>
**Step 1:** Install the necessary packages in your Python environment.
```bash
pip install opentelemetry-api \
opentelemetry-sdk \
opentelemetry-exporter-otlp \
'litellm[proxy]'
```
**Step 2:** Configure otel for the LiteLLM Proxy Server
Add the following to `config.yaml`:
```yaml
litellm_settings:
callbacks: ['otel']
```
**Step 3:** Set the following environment variables:
```bash
export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443"
export OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>"
export OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
export OTEL_TRACES_EXPORTER="otlp"
export OTEL_METRICS_EXPORTER="otlp"
export OTEL_LOGS_EXPORTER="otlp"
```
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
**Step 4:** Run the proxy server using the config file:
```bash
litellm --config config.yaml
```
Now any calls made through your LiteLLM proxy server will be traced and sent to SigNoz.
You should be able to view traces in Signoz Cloud under the traces tab:
![LiteLLM Proxy Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-traces.webp)
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
![LiteLLM Proxy Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-detailed-traces.webp)
## Dashboard
You can also check out our custom LiteLLM Proxy dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-proxy-dashboard/) which provides specialized visualizations for monitoring your LiteLLM Proxy usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
![LiteLLM Proxy Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-proxy-dashboard.webp)
</TabItem>
</Tabs>
docs/my-website/docs/observability/sumologic_integration.md
+45
View File
@@ -148,6 +148,51 @@ Example payload:
## Advanced Configuration
### Log Format
The Sumo Logic integration uses **NDJSON (newline-delimited JSON)** format by default. This format is optimal for Sumo Logic's parsing capabilities and allows Field Extraction Rules to work at ingest time.
#### NDJSON Format
Each log entry is sent as a separate line in the HTTP request:
```
{"id":"chatcmpl-1","model":"gpt-3.5-turbo","response_cost":0.0001,...}
{"id":"chatcmpl-2","model":"gpt-4","response_cost":0.0003,...}
{"id":"chatcmpl-3","model":"gpt-3.5-turbo","response_cost":0.0001,...}
```
#### Benefits for Field Extraction Rules (FERs)
With NDJSON format, you can create Field Extraction Rules directly:
```
_sourceCategory=litellm/logs
| json field=_raw "model", "response_cost", "user" as model, cost, user
```
**Before NDJSON** (with JSON array format):
- Required `parse regex ... multi` workaround
- FERs couldn't parse at ingest time
- Query-time parsing impacted dashboard performance
**After NDJSON**:
- ✅ FERs parse fields at ingest time
- ✅ No query-time workarounds needed
- ✅ Better dashboard performance
- ✅ Simpler query syntax
#### Changing the Log Format (Advanced)
If you need to change the log format (not recommended for Sumo Logic):
```yaml
callback_settings:
sumologic:
callback_type: generic_api
callback_name: sumologic
log_format: json_array # Override to use JSON array instead
```
### Batching Settings
Control how LiteLLM batches logs before sending to Sumo Logic:
docs/my-website/docs/providers/abliteration.md
+109
View File
@@ -0,0 +1,109 @@
# Abliteration
## Overview
| Property | Details |
|-------|-------|
| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. |
| Provider Route on LiteLLM | `abliteration/` |
| Link to Provider Doc | [Abliteration](https://abliteration.ai) |
| Base URL | `https://api.abliteration.ai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key
```
## Sample Usage
```python showLineNumbers title="Abliteration Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
)
print(response)
```
## Sample Usage - Streaming
```python showLineNumbers title="Abliteration Streaming Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Stream a short reply"}],
stream=True,
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy Server
1. Add the model to your proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: abliteration-chat
litellm_params:
model: abliteration/abliterated-model
api_key: os.environ/ABLITERATION_API_KEY
```
2. Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
## Direct API Usage (Bearer Token)
Use the environment variable as a Bearer token against the OpenAI-compatible endpoint:
`https://api.abliteration.ai/v1/chat/completions`.
```bash showLineNumbers title="cURL"
export ABLITERATION_API_KEY=""
curl https://api.abliteration.ai/v1/chat/completions \
-H "Authorization: Bearer ${ABLITERATION_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}]
}'
```
```python showLineNumbers title="Python (requests)"
import os
import requests
api_key = os.environ["ABLITERATION_API_KEY"]
response = requests.post(
"https://api.abliteration.ai/v1/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}],
},
timeout=60,
)
print(response.json())
```
docs/my-website/docs/providers/anthropic.md
+5 -3
View File
@@ -444,7 +444,7 @@ Here's what a sample Raw Request from LiteLLM for Anthropic Context Caching look
POST Request Sent from LiteLLM:
curl -X POST \
https://api.anthropic.com/v1/messages \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' -H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' \
-d '{'model': 'claude-3-5-sonnet-20240620', [
{
"role": "user",
@@ -472,6 +472,8 @@ https://api.anthropic.com/v1/messages \
"max_tokens": 10
}'
```
**Note:** Anthropic no longer requires the `anthropic-beta: prompt-caching-2024-07-31` header. Prompt caching now works automatically when you use `cache_control` in your messages.
:::
### Caching - Large Context Caching
@@ -1690,9 +1692,9 @@ Assistant:
```
## Usage - PDF
## Usage - PDF
Pass base64 encoded PDF files to Anthropic models using the `image_url` field.
Pass base64 encoded PDF files to Anthropic models using the `file` content type with a `file_data` field.
<Tabs>
<TabItem value="sdk" label="SDK">
docs/my-website/docs/providers/apertis.md
+129
View File
@@ -0,0 +1,129 @@
# Apertis AI (Stima API)
## Overview
| Property | Details |
|-------|-------|
| Description | Apertis AI (formerly Stima API) is a unified API platform providing access to 430+ AI models through a single interface, with cost savings of up to 50%. |
| Provider Route on LiteLLM | `apertis/` |
| Link to Provider Doc | [Apertis AI Website ↗](https://api.stima.tech) |
| Base URL | `https://api.stima.tech/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Apertis AI?
Apertis AI is a unified API platform that lets developers:
- **Access 430+ AI Models**: All models through a single API
- **Save 50% on Costs**: Competitive pricing with significant discounts
- **Unified Billing**: Single bill for all model usage
- **Quick Setup**: Start with just $2 registration
- **GitHub Integration**: Link with your GitHub account
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
```
Get your Apertis AI API key from [api.stima.tech](https://api.stima.tech).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Apertis AI Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Apertis AI call
response = completion(
model="apertis/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Apertis AI Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Apertis AI call with streaming
response = completion(
model="apertis/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export STIMA_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: apertis-model
litellm_params:
model: apertis/model-name # Replace with actual model name
api_key: os.environ/STIMA_API_KEY
```
## Supported OpenAI Parameters
Apertis AI supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 430+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
## Cost Benefits
Apertis AI offers significant cost advantages:
- **50% Cost Savings**: Save money compared to direct provider costs
- **Unified Billing**: Single invoice for all your AI model usage
- **Low Entry**: Start with just $2 registration
## Model Availability
With access to 430+ AI models, Apertis AI provides:
- Multiple providers through one API
- Latest model releases
- Various model types (text, image, video)
## Additional Resources
- [Apertis AI Website](https://api.stima.tech)
- [Apertis AI Enterprise](https://api.stima.tech/enterprise)
docs/my-website/docs/providers/aws_polly.md
+364
View File
@@ -0,0 +1,364 @@
# AWS Polly Text to Speech (tts)
## Overview
| Property | Details |
|-------|-------|
| Description | Convert text to natural-sounding speech using AWS Polly's neural and standard TTS engines |
| Provider Route on LiteLLM | `aws_polly/` |
| Supported Operations | `/audio/speech` |
| Link to Provider Doc | [AWS Polly SynthesizeSpeech ↗](https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html) |
## Quick Start
### **LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
import litellm
from pathlib import Path
import os
# Set environment variables
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = "us-east-1"
# AWS Polly call
speech_file_path = Path(__file__).parent / "speech.mp3"
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="the quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: polly-neural
litellm_params:
model: aws_polly/neural
aws_access_key_id: "os.environ/AWS_ACCESS_KEY_ID"
aws_secret_access_key: "os.environ/AWS_SECRET_ACCESS_KEY"
aws_region_name: "us-east-1"
```
## Polly Engines
AWS Polly supports different speech synthesis engines. Specify the engine in the model name:
| Model | Engine | Cost (per 1M chars) | Description |
|-------|--------|---------------------|-------------|
| `aws_polly/standard` | Standard | $4.00 | Original Polly voices, faster and lowest cost |
| `aws_polly/neural` | Neural | $16.00 | More natural, human-like speech (recommended) |
| `aws_polly/generative` | Generative | $30.00 | Most expressive, highest quality (limited voices) |
| `aws_polly/long-form` | Long-form | $100.00 | Optimized for long content like articles |
### **LiteLLM SDK**
```python showLineNumbers title="Using Different Engines"
import litellm
# Neural engine (recommended)
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello world",
)
# Standard engine (lower cost)
response = litellm.speech(
model="aws_polly/standard",
voice="Joanna",
input="Hello world",
)
# Generative engine (highest quality)
response = litellm.speech(
model="aws_polly/generative",
voice="Matthew",
input="Hello world",
)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: polly-neural
litellm_params:
model: aws_polly/neural
aws_region_name: "us-east-1"
- model_name: polly-standard
litellm_params:
model: aws_polly/standard
aws_region_name: "us-east-1"
- model_name: polly-generative
litellm_params:
model: aws_polly/generative
aws_region_name: "us-east-1"
```
## Available Voices
### Native Polly Voices
AWS Polly has many voices across different languages. Here are popular US English voices:
| Voice | Gender | Engine Support |
|-------|--------|----------------|
| `Joanna` | Female | Neural, Standard |
| `Matthew` | Male | Neural, Standard, Generative |
| `Ivy` | Female (child) | Neural, Standard |
| `Kendra` | Female | Neural, Standard |
| `Amy` | Female (British) | Neural, Standard |
| `Brian` | Male (British) | Neural, Standard |
### **LiteLLM SDK**
```python showLineNumbers title="Using Native Polly Voices"
import litellm
# US English female
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello from Joanna",
)
# US English male
response = litellm.speech(
model="aws_polly/neural",
voice="Matthew",
input="Hello from Matthew",
)
# British English female
response = litellm.speech(
model="aws_polly/neural",
voice="Amy",
input="Hello from Amy",
)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: polly-joanna
litellm_params:
model: aws_polly/neural
voice: "Joanna"
aws_region_name: "us-east-1"
- model_name: polly-matthew
litellm_params:
model: aws_polly/neural
voice: "Matthew"
aws_region_name: "us-east-1"
```
### OpenAI Voice Mappings
LiteLLM also supports OpenAI voice names, which are automatically mapped to Polly voices:
| OpenAI Voice | Maps to Polly Voice |
|--------------|---------------------|
| `alloy` | Joanna |
| `echo` | Matthew |
| `fable` | Amy |
| `onyx` | Brian |
| `nova` | Ivy |
| `shimmer` | Kendra |
### **LiteLLM SDK**
```python showLineNumbers title="Using OpenAI Voice Names"
import litellm
# These are equivalent
response = litellm.speech(
model="aws_polly/neural",
voice="alloy", # Maps to Joanna
input="Hello world",
)
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna", # Native Polly voice
input="Hello world",
)
```
## SSML Support
AWS Polly supports SSML (Speech Synthesis Markup Language) for advanced control over speech output. LiteLLM automatically detects SSML input.
### **LiteLLM SDK**
```python showLineNumbers title="SSML Example"
import litellm
ssml_input = """
<speak>
Hello, <break time="500ms"/>
this is a test with <emphasis level="strong">emphasis</emphasis>
and <prosody rate="slow">slower speech</prosody>.
</speak>
"""
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input=ssml_input,
)
```
### **LiteLLM PROXY**
```bash showLineNumbers title="cURL Request with SSML"
curl -X POST http://localhost:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "polly-neural",
"voice": "Joanna",
"input": "<speak>Hello <break time=\"500ms\"/> world</speak>"
}' \
--output speech.mp3
```
## Supported Parameters
```python showLineNumbers title="All Parameters"
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna", # Required: Voice selection
input="text to convert", # Required: Input text (or SSML)
response_format="mp3", # Optional: mp3, ogg_vorbis, pcm
# AWS-specific parameters
language_code="en-US", # Optional: Language code
sample_rate="22050", # Optional: Sample rate in Hz
)
```
## Response Formats
| Format | Description |
|--------|-------------|
| `mp3` | MP3 audio (default) |
| `ogg_vorbis` | Ogg Vorbis audio |
| `pcm` | Raw PCM audio |
### **LiteLLM SDK**
```python showLineNumbers title="Different Response Formats"
import litellm
# MP3 (default)
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello",
response_format="mp3",
)
# Ogg Vorbis
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello",
response_format="ogg_vorbis",
)
```
## AWS Authentication
LiteLLM supports multiple AWS authentication methods.
### **LiteLLM SDK**
```python showLineNumbers title="Authentication Options"
import litellm
import os
# Option 1: Environment variables (recommended)
os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key"
os.environ["AWS_REGION_NAME"] = "us-east-1"
response = litellm.speech(model="aws_polly/neural", voice="Joanna", input="Hello")
# Option 2: Pass credentials directly
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello",
aws_access_key_id="your-access-key",
aws_secret_access_key="your-secret-key",
aws_region_name="us-east-1",
)
# Option 3: IAM Role (when running on AWS)
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello",
aws_region_name="us-east-1",
)
# Option 4: AWS Profile
response = litellm.speech(
model="aws_polly/neural",
voice="Joanna",
input="Hello",
aws_profile_name="my-profile",
)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
# Using environment variables
- model_name: polly-neural
litellm_params:
model: aws_polly/neural
aws_access_key_id: "os.environ/AWS_ACCESS_KEY_ID"
aws_secret_access_key: "os.environ/AWS_SECRET_ACCESS_KEY"
aws_region_name: "us-east-1"
# Using IAM Role (when proxy runs on AWS)
- model_name: polly-neural-iam
litellm_params:
model: aws_polly/neural
aws_region_name: "us-east-1"
# Using AWS Profile
- model_name: polly-neural-profile
litellm_params:
model: aws_polly/neural
aws_profile_name: "my-profile"
```
## Async Support
```python showLineNumbers title="Async Usage"
import litellm
import asyncio
async def main():
response = await litellm.aspeech(
model="aws_polly/neural",
voice="Joanna",
input="Hello from async AWS Polly",
aws_region_name="us-east-1",
)
with open("output.mp3", "wb") as f:
f.write(response.content)
asyncio.run(main())
```
docs/my-website/docs/providers/azure_ai/azure_model_router.md
+232
View File
@@ -0,0 +1,232 @@
# Azure Model Router
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
## LiteLLM Python SDK
### Basic Usage
```python
import litellm
import os
response = litellm.completion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
)
print(response)
```
### Streaming with Usage Tracking
```python
import litellm
import os
response = await litellm.acompletion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "hi"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
stream=True,
stream_options={"include_usage": True},
)
async for chunk in response:
print(chunk)
```
## LiteLLM Proxy (AI Gateway)
### config.yaml
```yaml
model_list:
- model_name: azure-model-router
litellm_params:
model: azure_ai/azure-model-router
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
```
### Start Proxy
```bash
litellm --config config.yaml
```
### Test Request
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "azure-model-router",
"messages": [{"role": "user", "content": "Hello!"}]
}'
```
## Add Azure Model Router via LiteLLM UI
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
### Select Provider
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
#### Navigate to Models Page
![Navigate to Models](./img/azure_model_router_01.jpeg)
#### Click Provider Dropdown
![Click Provider](./img/azure_model_router_02.jpeg)
#### Choose Azure AI Foundry
![Select Azure AI Foundry](./img/azure_model_router_03.jpeg)
### Configure Model Name
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
#### Click Model Name Field
![Click Model Field](./img/azure_model_router_04.jpeg)
#### Select Custom Model Name
![Select Custom Model](./img/azure_model_router_05.jpeg)
#### Enter LiteLLM Model Name
![LiteLLM Model Name](./img/azure_model_router_06.jpeg)
#### Click Custom Model Name Field
![Enter Custom Name Field](./img/azure_model_router_07.jpeg)
#### Type Model Prefix
Type `azure_ai/` as the prefix.
![Type azure_ai prefix](./img/azure_model_router_08.jpeg)
#### Copy Model Name from Azure Portal
Switch to Azure AI Foundry and copy your model router deployment name.
![Azure Portal Model Name](./img/azure_model_router_09.jpeg)
![Copy Model Name](./img/azure_model_router_10.jpeg)
#### Paste Model Name
Paste to get `azure_ai/azure-model-router`.
![Paste Model Name](./img/azure_model_router_11.jpeg)
### Configure API Base and Key
Copy the endpoint URL and API key from Azure portal.
#### Copy API Base URL from Azure
![Copy API Base](./img/azure_model_router_12.jpeg)
#### Enter API Base in LiteLLM
![Click API Base Field](./img/azure_model_router_13.jpeg)
![Paste API Base](./img/azure_model_router_14.jpeg)
#### Copy API Key from Azure
![Copy API Key](./img/azure_model_router_15.jpeg)
#### Enter API Key in LiteLLM
![Enter API Key](./img/azure_model_router_16.jpeg)
### Test and Add Model
Verify your configuration works and save the model.
#### Test Connection
![Test Connection](./img/azure_model_router_17.jpeg)
#### Close Test Dialog
![Close Dialog](./img/azure_model_router_18.jpeg)
#### Add Model
![Add Model](./img/azure_model_router_19.jpeg)
### Verify in Playground
Test your model and verify cost tracking is working.
#### Open Playground
![Go to Playground](./img/azure_model_router_20.jpeg)
#### Select Model
![Select Model](./img/azure_model_router_21.jpeg)
#### Send Test Message
![Send Message](./img/azure_model_router_22.jpeg)
#### View Logs
![View Logs](./img/azure_model_router_23.jpeg)
#### Verify Cost Tracking
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
![Verify Cost](./img/azure_model_router_24.jpeg)
## Cost Tracking
LiteLLM automatically handles cost tracking for Azure Model Router by:
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### Example Response with Cost
```python
import litellm
response = litellm.completion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
# The response will show the actual model used
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
# Get cost
from litellm import completion_cost
cost = completion_cost(completion_response=response)
print(f"Cost: ${cost}")
```
docs/my-website/docs/providers/azure_ai/img/azure_model_router_01.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 254 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_02.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 240 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_03.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 239 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_04.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 247 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_05.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 272 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_06.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 307 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_07.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 248 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_08.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 315 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_09.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 492 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_10.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 562 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_11.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 265 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_12.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 471 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_13.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 225 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_14.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 219 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_15.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 485 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_16.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 227 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_17.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 236 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_18.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 214 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_19.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 232 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_20.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 256 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_21.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 250 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_22.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 206 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_23.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 311 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_24.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 419 KiB

docs/my-website/docs/providers/azure_ai_img.md
+135 -2
View File
@@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Generation
# Azure AI Image Generation (Black Forest Labs - Flux)
Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
@@ -12,7 +12,7 @@ Azure AI provides powerful image generation capabilities using FLUX models from
| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/generations`](#image-generation) |
| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-editing) |
## Setup
@@ -33,6 +33,7 @@ Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
|------------|-------------|----------------|
| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
| `azure_ai/flux.2-pro` | FLUX 2 Pro model for next-generation image generation | $0.04 |
## Image Generation
@@ -85,6 +86,32 @@ print(response.data[0].url)
</TabItem>
<TabItem value="flux2" label="FLUX 2 Pro">
```python showLineNumbers title="FLUX 2 Pro Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com
# Generate image with FLUX 2 Pro
response = litellm.image_generation(
model="azure_ai/flux.2-pro",
prompt="A photograph of a red fox in an autumn forest",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
size="1024x1024",
n=1
)
print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
@@ -165,6 +192,15 @@ model_list:
model_info:
mode: image_generation
- model_name: azure-flux-2-pro
litellm_params:
model: azure_ai/flux.2-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
api_version: preview
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
@@ -239,6 +275,103 @@ curl --location 'http://localhost:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Image Editing
FLUX 2 Pro supports image editing by passing an input image along with a prompt describing the desired modifications.
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic-edit" label="Basic Image Edit">
```python showLineNumbers title="Basic Image Editing with FLUX 2 Pro"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://litellm-ci-cd-prod.services.ai.azure.com
# Edit an existing image
response = litellm.image_edit(
model="azure_ai/flux.2-pro",
prompt="Add a red hat to the subject",
image=open("input_image.png", "rb"),
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
)
print(response.data[0].b64_json) # FLUX 2 returns base64 encoded images
```
</TabItem>
<TabItem value="async-edit" label="Async Image Edit">
```python showLineNumbers title="Async Image Editing"
import litellm
import asyncio
import os
async def edit_image():
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
response = await litellm.aimage_edit(
model="azure_ai/flux.2-pro",
prompt="Change the background to a sunset beach",
image=open("input_image.png", "rb"),
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version="preview",
)
return response
asyncio.run(edit_image())
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
<Tabs>
<TabItem value="curl-edit" label="cURL">
```bash showLineNumbers title="Image Edit via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/edits' \
--header 'Authorization: Bearer sk-1234' \
--form 'model="azure-flux-2-pro"' \
--form 'prompt="Add sunglasses to the person"' \
--form 'image=@"input_image.png"'
```
</TabItem>
<TabItem value="openai-sdk-edit" label="OpenAI SDK">
```python showLineNumbers title="Image Edit via Proxy - OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="sk-1234"
)
response = client.images.edit(
model="azure-flux-2-pro",
prompt="Make the sky more dramatic with storm clouds",
image=open("input_image.png", "rb"),
)
print(response.data[0].b64_json)
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Generation supports the following OpenAI-compatible parameters:
docs/my-website/docs/providers/bedrock.md
+73 -1
View File
@@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Rerank Endpoint | `/rerank` |
@@ -967,6 +967,30 @@ Control the processing tier for your Bedrock requests using `serviceTier`. Valid
[Bedrock ServiceTier API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ServiceTier.html)
### OpenAI-compatible `service_tier` parameter
LiteLLM also supports the OpenAI-style `service_tier` parameter, which is automatically translated to Bedrock's native `serviceTier` format:
| OpenAI `service_tier` | Bedrock `serviceTier` |
|-----------------------|----------------------|
| `"priority"` | `{"type": "priority"}` |
| `"default"` | `{"type": "default"}` |
| `"flex"` | `{"type": "flex"}` |
| `"auto"` | `{"type": "default"}` |
```python
from litellm import completion
# Using OpenAI-style service_tier parameter
response = completion(
model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
service_tier="priority" # Automatically translated to serviceTier={"type": "priority"}
)
```
### Native Bedrock `serviceTier` parameter
<Tabs>
<TabItem value="sdk" label="SDK">
@@ -1941,6 +1965,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Moonshot Kimi K2 Thinking | `completion(model='bedrock/moonshot.kimi-k2-thinking', messages=messages)` or `completion(model='bedrock/invoke/moonshot.kimi-k2-thinking', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding
@@ -2208,6 +2233,53 @@ response = completion(
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
### IAM Roles Anywhere (On-Premise / External Workloads)
[IAM Roles Anywhere](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/introduction.html) extends IAM roles to workloads **outside of AWS** (on-premise servers, edge devices, other clouds). It uses the same STS mechanism as regular IAM roles but authenticates via X.509 certificates instead of AWS credentials.
**Setup**: Configure the [AWS Signing Helper](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/credential-helper.html) as a credential process in `~/.aws/config`:
```ini
[profile litellm-roles-anywhere]
credential_process = aws_signing_helper credential-process \
--certificate /path/to/certificate.pem \
--private-key /path/to/private-key.pem \
--trust-anchor-arn arn:aws:rolesanywhere:us-east-1:123456789012:trust-anchor/abc123 \
--profile-arn arn:aws:rolesanywhere:us-east-1:123456789012:profile/def456 \
--role-arn arn:aws:iam::123456789012:role/MyBedrockRole
```
**Usage**: Reference the profile in LiteLLM:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
aws_profile_name="litellm-roles-anywhere",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
aws_profile_name: "litellm-roles-anywhere"
```
</TabItem>
</Tabs>
See the [IAM Roles Anywhere Getting Started Guide](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/getting-started.html) for trust anchor and profile setup.
Make the bedrock completion call
docs/my-website/docs/providers/bedrock_agentcore.md
+6
View File
@@ -11,6 +11,12 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
| Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` |
| Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) |
:::info
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
:::
## Quick Start
### Model Format to LiteLLM
docs/my-website/docs/providers/bedrock_embedding.md
+119
View File
@@ -172,6 +172,125 @@ print(f"Results available at: {output_s3_uri}")
**Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors.
## Amazon Nova Multimodal Embeddings
Amazon Nova supports multimodal embeddings for text, images, video, and audio. It offers flexible embedding dimensions and purposes optimized for different use cases.
### Supported Features
- **Modalities**: Text, Image, Video, Audio
- **Dimensions**: 256, 384, 1024, 3072 (default: 3072)
- **Embedding Purposes**:
- `GENERIC_INDEX` (default)
- `GENERIC_RETRIEVAL`
- `TEXT_RETRIEVAL`
- `IMAGE_RETRIEVAL`
- `VIDEO_RETRIEVAL`
- `AUDIO_RETRIEVAL`
- `CLASSIFICATION`
- `CLUSTERING`
### Text Embedding
```python
from litellm import embedding
response = embedding(
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
input=["Hello, world!"],
aws_region_name="us-east-1",
dimensions=1024, # Optional: 256, 384, 1024, or 3072
)
print(response.data[0].embedding)
```
### Image Embedding with Base64
Amazon Nova accepts images in base64 format using the standard data URL format:
```python
import base64
from litellm import embedding
# Method 1: Load image from file
with open("image.jpg", "rb") as image_file:
image_data = base64.b64encode(image_file.read()).decode('utf-8')
# Create data URL with proper format
image_base64 = f"data:image/jpeg;base64,{image_data}"
response = embedding(
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
input=[image_base64],
aws_region_name="us-east-1",
dimensions=1024,
)
print(f"Image embedding: {response.data[0].embedding[:10]}...") # First 10 dimensions
```
#### Supported Image Formats
Nova supports the following image formats:
- JPEG: `data:image/jpeg;base64,...`
- PNG: `data:image/png;base64,...`
- GIF: `data:image/gif;base64,...`
- WebP: `data:image/webp;base64,...`
#### Complete Example with Error Handling
```python
import base64
from litellm import embedding
def get_image_embedding(image_path, dimensions=1024):
"""
Get embedding for an image file.
Args:
image_path: Path to the image file
dimensions: Embedding dimension (256, 384, 1024, or 3072)
Returns:
List of embedding values
"""
try:
# Determine image format from file extension
if image_path.lower().endswith('.png'):
mime_type = "image/png"
elif image_path.lower().endswith(('.jpg', '.jpeg')):
mime_type = "image/jpeg"
elif image_path.lower().endswith('.gif'):
mime_type = "image/gif"
elif image_path.lower().endswith('.webp'):
mime_type = "image/webp"
else:
raise ValueError(f"Unsupported image format: {image_path}")
# Read and encode image
with open(image_path, "rb") as image_file:
image_data = base64.b64encode(image_file.read()).decode('utf-8')
image_base64 = f"data:{mime_type};base64,{image_data}"
# Get embedding
response = embedding(
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
input=[image_base64],
aws_region_name="us-east-1",
dimensions=dimensions,
)
return response.data[0].embedding
except Exception as e:
print(f"Error getting image embedding: {e}")
raise
# Example usage
image_embedding = get_image_embedding("photo.jpg", dimensions=1024)
print(f"Got embedding with {len(image_embedding)} dimensions")
```
### Error Handling
#### Common Errors
docs/my-website/docs/providers/bedrock_imported.md
+177 -1
View File
@@ -431,4 +431,180 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"max_tokens": 300,
"temperature": 0.5
}'
```
```
### Moonshot Kimi K2 Thinking
Moonshot AI's Kimi K2 Thinking model is now available on Amazon Bedrock. This model features advanced reasoning capabilities with automatic reasoning content extraction.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/moonshot.kimi-k2-thinking`, `bedrock/invoke/moonshot.kimi-k2-thinking` |
| Provider Documentation | [AWS Bedrock Moonshot Announcement ↗](https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/) |
| Supported Parameters | `temperature`, `max_tokens`, `top_p`, `stream`, `tools`, `tool_choice` |
| Special Features | Reasoning content extraction, Tool calling |
#### Supported Features
- **Reasoning Content Extraction**: Automatically extracts `<reasoning>` tags and returns them as `reasoning_content` (similar to OpenAI's o1 models)
- **Tool Calling**: Full support for function/tool calling with tool responses
- **Streaming**: Both streaming and non-streaming responses
- **System Messages**: System message support
#### Basic Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python title="Moonshot Kimi K2 SDK Usage" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2" # or your preferred region
# Basic completion
response = completion(
model="bedrock/moonshot.kimi-k2-thinking", # or bedrock/invoke/moonshot.kimi-k2-thinking
messages=[
{"role": "user", "content": "What is 2+2? Think step by step."}
],
temperature=0.7,
max_tokens=200
)
print(response.choices[0].message.content)
# Access reasoning content if present
if response.choices[0].message.reasoning_content:
print("Reasoning:", response.choices[0].message.reasoning_content)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: kimi-k2
litellm_params:
model: bedrock/moonshot.kimi-k2-thinking
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
```
**2. Start proxy**
```bash title="Start LiteLLM Proxy" showLineNumbers
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash title="Test Kimi K2 via Proxy" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "kimi-k2",
"messages": [
{
"role": "user",
"content": "What is 2+2? Think step by step."
}
],
"temperature": 0.7,
"max_tokens": 200
}'
```
</TabItem>
</Tabs>
#### Tool Calling Example
```python title="Kimi K2 with Tool Calling" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
# Tool calling example
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "What's the weather in Tokyo?"}
],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name"
}
},
"required": ["location"]
}
}
}
]
)
if response.choices[0].message.tool_calls:
tool_call = response.choices[0].message.tool_calls[0]
print(f"Tool called: {tool_call.function.name}")
print(f"Arguments: {tool_call.function.arguments}")
```
#### Streaming Example
```python title="Kimi K2 Streaming" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "Explain quantum computing in simple terms."}
],
stream=True,
temperature=0.7
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
# Check for reasoning content in streaming
if hasattr(chunk.choices[0].delta, 'reasoning_content') and chunk.choices[0].delta.reasoning_content:
print(f"\n[Reasoning: {chunk.choices[0].delta.reasoning_content}]")
```
#### Supported Parameters
| Parameter | Type | Description | Supported |
|-----------|------|-------------|-----------|
| `temperature` | float (0-1) | Controls randomness in output | ✅ |
| `max_tokens` | integer | Maximum tokens to generate | ✅ |
| `top_p` | float | Nucleus sampling parameter | ✅ |
| `stream` | boolean | Enable streaming responses | ✅ |
| `tools` | array | Tool/function definitions | ✅ |
| `tool_choice` | string/object | Tool choice specification | ✅ |
| `stop` | array | Stop sequences | ❌ (Not supported on Bedrock) |
docs/my-website/docs/providers/chutes.md
+172
View File
@@ -0,0 +1,172 @@
# Chutes
## Overview
| Property | Details |
|-------|-------|
| Description | Chutes is a cloud-native AI deployment platform that allows you to deploy, run, and scale LLM applications with OpenAI-compatible APIs using pre-built templates for popular frameworks like vLLM and SGLang. |
| Provider Route on LiteLLM | `chutes/` |
| Link to Provider Doc | [Chutes Website ↗](https://chutes.ai) |
| Base URL | `https://llm.chutes.ai/v1/` |
| Supported Operations | [`/chat/completions`](#sample-usage), Embeddings |
<br />
## What is Chutes?
Chutes is a powerful AI deployment and serving platform that provides:
- **Pre-built Templates**: Ready-to-use configurations for vLLM, SGLang, diffusion models, and embeddings
- **OpenAI-Compatible APIs**: Use standard OpenAI SDKs and clients
- **Multi-GPU Scaling**: Support for large models across multiple GPUs
- **Streaming Responses**: Real-time model outputs
- **Custom Configurations**: Override any parameter for your specific needs
- **Performance Optimization**: Pre-configured optimization settings
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
```
Get your Chutes API key from [chutes.ai](https://chutes.ai).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Chutes Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Chutes call
response = completion(
model="chutes/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Chutes Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Chutes call with streaming
response = completion(
model="chutes/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export CHUTES_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: chutes-model
litellm_params:
model: chutes/model-name # Replace with actual model name
api_key: os.environ/CHUTES_API_KEY
```
## Supported OpenAI Parameters
Chutes supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID or HuggingFace model identifier |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
## Support Frameworks
Chutes provides optimized templates for popular AI frameworks:
### vLLM (High-Performance LLM Serving)
- OpenAI-compatible endpoints
- Multi-GPU scaling support
- Advanced optimization settings
- Best for production workloads
### SGLang (Advanced LLM Serving)
- Structured generation capabilities
- Advanced features and controls
- Custom configuration options
- Best for complex use cases
### Diffusion Models (Image Generation)
- Pre-configured image generation templates
- Optimized settings for best results
- Support for popular diffusion models
### Embedding Models
- Text embedding templates
- Vector search optimization
- Support for popular embedding models
## Authentication
Chutes supports multiple authentication methods:
- API Key via `X-API-Key` header
- Bearer token via `Authorization` header
Example for LiteLLM (uses environment variable):
```python
os.environ["CHUTES_API_KEY"] = "your-api-key"
```
## Performance Optimization
Chutes offers hardware selection and optimization:
- **Small Models (7B-13B)**: 1 GPU with 24GB VRAM
- **Medium Models (30B-70B)**: 4 GPUs with 80GB VRAM each
- **Large Models (100B+)**: 8 GPUs with 140GB+ VRAM each
Engine optimization parameters available for fine-tuning performance.
## Deployment Options
Chutes provides flexible deployment:
- **Quick Setup**: Use pre-built templates for instant deployment
- **Custom Images**: Deploy with custom Docker images
- **Scaling**: Configure max instances and auto-scaling thresholds
- **Hardware**: Choose specific GPU types and configurations
## Additional Resources
- [Chutes Documentation](https://chutes.ai/docs)
- [Chutes Getting Started](https://chutes.ai/docs/getting-started/running-a-chute)
- [Chutes API Reference](https://chutes.ai/docs/sdk-reference)
docs/my-website/docs/providers/databricks.md
+94
View File
@@ -11,6 +11,99 @@ LiteLLM supports all models on Databricks
:::
## Authentication
LiteLLM supports multiple authentication methods for Databricks, listed in order of preference:
### OAuth M2M (Recommended for Production)
OAuth Machine-to-Machine authentication using Service Principal credentials is the **recommended method for production** deployments per Databricks Partner requirements.
```python
import os
from litellm import completion
# Set OAuth credentials (Service Principal)
os.environ["DATABRICKS_CLIENT_ID"] = "your-service-principal-application-id"
os.environ["DATABRICKS_CLIENT_SECRET"] = "your-service-principal-secret"
os.environ["DATABRICKS_API_BASE"] = "https://adb-xxx.azuredatabricks.net/serving-endpoints"
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
### Personal Access Token (PAT)
PAT authentication is supported for development and testing scenarios.
```python
import os
from litellm import completion
os.environ["DATABRICKS_API_KEY"] = "dapi..." # Your Personal Access Token
os.environ["DATABRICKS_API_BASE"] = "https://adb-xxx.azuredatabricks.net/serving-endpoints"
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
### Databricks SDK Authentication (Automatic)
If no credentials are provided, LiteLLM will use the Databricks SDK for automatic authentication. This supports OAuth, Azure AD, and other unified auth methods configured in your environment.
```python
from litellm import completion
# No environment variables needed - uses Databricks SDK unified auth
# Requires: pip install databricks-sdk
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
## Custom User-Agent for Partner Attribution
If you're building a product on top of LiteLLM that integrates with Databricks, you can pass your own partner identifier for proper attribution in Databricks telemetry.
The partner name will be prefixed to the LiteLLM user agent:
```python
# Via parameter
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
user_agent="mycompany/1.0.0",
)
# Resulting User-Agent: mycompany_litellm/1.79.1
# Via environment variable
os.environ["DATABRICKS_USER_AGENT"] = "mycompany/1.0.0"
# Resulting User-Agent: mycompany_litellm/1.79.1
```
| Input | Resulting User-Agent |
|-------|---------------------|
| (none) | `litellm/1.79.1` |
| `mycompany/1.0.0` | `mycompany_litellm/1.79.1` |
| `partner_product/2.5.0` | `partner_product_litellm/1.79.1` |
| `acme` | `acme_litellm/1.79.1` |
**Note:** The version from your custom user agent is ignored; LiteLLM's version is always used.
## Security
LiteLLM automatically redacts sensitive information (tokens, secrets, API keys) from all debug logs to prevent credential leakage. This includes:
- Authorization headers
- API keys and tokens
- Client secrets
- Personal access tokens (PATs)
## Usage
<Tabs>
@@ -51,6 +144,7 @@ response = completion(
model: databricks/databricks-dbrx-instruct
api_key: os.environ/DATABRICKS_API_KEY
api_base: os.environ/DATABRICKS_API_BASE
user_agent: "mycompany/1.0.0" # Optional: for partner attribution
```
docs/my-website/docs/providers/gigachat.md
+283
View File
@@ -0,0 +1,283 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# GigaChat
https://developers.sber.ru/docs/ru/gigachat/api/overview
GigaChat is Sber AI's large language model, Russia's leading LLM provider.
:::tip
**We support ALL GigaChat models, just set `model=gigachat/<any-model-on-gigachat>` as a prefix when sending litellm requests**
:::
:::warning
GigaChat API uses self-signed SSL certificates. You must pass `ssl_verify=False` in your requests.
:::
## Supported Features
| Feature | Supported |
|---------|-----------|
| Chat Completion | Yes |
| Streaming | Yes |
| Async | Yes |
| Function Calling / Tools | Yes |
| Structured Output (JSON Schema) | Yes (via function call emulation) |
| Image Input | Yes (base64 and URL) - GigaChat-2-Max, GigaChat-2-Pro only |
| Embeddings | Yes |
## API Key
GigaChat uses OAuth authentication. Set your credentials as environment variables:
```python
import os
# Required: Set credentials (base64-encoded client_id:client_secret)
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
# Optional: Set scope (default is GIGACHAT_API_PERS for personal use)
os.environ['GIGACHAT_SCOPE'] = "GIGACHAT_API_PERS" # or GIGACHAT_API_B2B for business
```
Get your credentials at: https://developers.sber.ru/studio/
## Sample Usage
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[
{"role": "user", "content": "Hello from LiteLLM!"}
],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[
{"role": "user", "content": "Hello from LiteLLM!"}
],
stream=True,
ssl_verify=False, # Required for GigaChat
)
for chunk in response:
print(chunk)
```
## Sample Usage - Function Calling
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string", "description": "City name"}
},
"required": ["city"]
}
}
}]
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[{"role": "user", "content": "What's the weather in Moscow?"}],
tools=tools,
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Structured Output
GigaChat supports structured output via JSON schema (emulated through function calling):
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max",
messages=[{"role": "user", "content": "Extract info: John is 30 years old"}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "person",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
}
}
}
},
ssl_verify=False, # Required for GigaChat
)
print(response) # Returns JSON: {"name": "John", "age": 30}
```
## Sample Usage - Image Input
GigaChat supports image input via base64 or URL (GigaChat-2-Max and GigaChat-2-Pro only):
```python
from litellm import completion
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = completion(
model="gigachat/GigaChat-2-Max", # Vision requires GigaChat-2-Max or GigaChat-2-Pro
messages=[{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]
}],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Sample Usage - Embeddings
```python
from litellm import embedding
import os
os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here"
response = embedding(
model="gigachat/Embeddings",
input=["Hello world", "How are you?"],
ssl_verify=False, # Required for GigaChat
)
print(response)
```
## Usage with LiteLLM Proxy
### 1. Set GigaChat Models on config.yaml
```yaml
model_list:
- model_name: gigachat
litellm_params:
model: gigachat/GigaChat-2-Max
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
- model_name: gigachat-lite
litellm_params:
model: gigachat/GigaChat-2-Lite
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
- model_name: gigachat-embeddings
litellm_params:
model: gigachat/Embeddings
api_key: "os.environ/GIGACHAT_CREDENTIALS"
ssl_verify: false
```
### 2. Start Proxy
```bash
litellm --config config.yaml
```
### 3. Test it
<Tabs>
<TabItem value="Curl" label="Curl Request">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gigachat",
"messages": [
{
"role": "user",
"content": "Hello!"
}
]
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gigachat",
messages=[{"role": "user", "content": "Hello!"}]
)
print(response)
```
</TabItem>
</Tabs>
## Supported Models
### Chat Models
| Model Name | Context Window | Vision | Description |
|------------|----------------|--------|-------------|
| gigachat/GigaChat-2-Lite | 128K | No | Fast, lightweight model |
| gigachat/GigaChat-2-Pro | 128K | Yes | Professional model with vision |
| gigachat/GigaChat-2-Max | 128K | Yes | Maximum capability model |
### Embedding Models
| Model Name | Max Input | Dimensions | Description |
|------------|-----------|------------|-------------|
| gigachat/Embeddings | 512 | 1024 | Standard embeddings |
| gigachat/Embeddings-2 | 512 | 1024 | Updated embeddings |
| gigachat/EmbeddingsGigaR | 4096 | 2560 | High-dimensional embeddings |
:::note
Available models may vary depending on your API access level (personal or business).
:::
## Limitations
- Only one function call per request (GigaChat API limitation)
- Maximum 1 image per message, 10 images total per conversation
- GigaChat API uses self-signed SSL certificates - `ssl_verify=False` is required
docs/my-website/docs/providers/groq.md
+14 -17
View File
@@ -150,15 +150,15 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| Model Name | Usage |
|--------------------|---------------------------------------------------------|
| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` |
| llama-3.1-70b-versatile | `completion(model="groq/llama-3.1-70b-versatile", messages)` |
| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` |
| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` |
| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
| llama-3.3-70b-versatile | `completion(model="groq/llama-3.3-70b-versatile", messages)` |
| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` |
| meta-llama/llama-4-scout-17b-16e-instruct | `completion(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", messages)` |
| meta-llama/llama-4-maverick-17b-128e-instruct | `completion(model="groq/meta-llama/llama-4-maverick-17b-128e-instruct", messages)` |
| meta-llama/llama-guard-4-12b | `completion(model="groq/meta-llama/llama-guard-4-12b", messages)` |
| qwen/qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
## Groq - Tool / Function Calling Example
@@ -261,31 +261,28 @@ if tool_calls:
print("second response\n", second_response)
```
## Groq - Vision Example
## Groq - Vision Example
Select Groq models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details.
Groq's Llama 4 models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
import os
from litellm import completion
os.environ["GROQ_API_KEY"] = "your-api-key"
# openai call
response = completion(
model = "groq/llama-3.2-11b-vision-preview",
model = "groq/meta-llama/llama-4-scout-17b-16e-instruct",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What’s in this image?"
"text": "What's in this image?"
},
{
"type": "image_url",
docs/my-website/docs/providers/llamagate.md
+228
View File
@@ -0,0 +1,228 @@
# LlamaGate
## Overview
| Property | Details |
|-------|-------|
| Description | LlamaGate is an OpenAI-compatible API gateway for open-source LLMs with credit-based billing. Access 26+ open-source models including Llama, Mistral, DeepSeek, and Qwen at competitive prices. |
| Provider Route on LiteLLM | `llamagate/` |
| Link to Provider Doc | [LlamaGate Documentation ↗](https://llamagate.dev/docs) |
| Base URL | `https://api.llamagate.dev/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage), [`/embeddings`](#embeddings) |
<br />
## What is LlamaGate?
LlamaGate provides access to open-source LLMs through an OpenAI-compatible API:
- **26+ Open-Source Models**: Llama 3.1/3.2, Mistral, Qwen, DeepSeek R1, and more
- **OpenAI-Compatible API**: Drop-in replacement for OpenAI SDK
- **Vision Models**: Qwen VL, LLaVA, olmOCR, UI-TARS for multimodal tasks
- **Reasoning Models**: DeepSeek R1, OpenThinker for complex problem-solving
- **Code Models**: CodeLlama, DeepSeek Coder, Qwen Coder, StarCoder2
- **Embedding Models**: Nomic, Qwen3 Embedding for RAG and search
- **Competitive Pricing**: $0.02-$0.55 per 1M tokens
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
```
Get your API key from [llamagate.dev](https://llamagate.dev).
## Supported Models
### General Purpose
| Model | Model ID |
|-------|----------|
| Llama 3.1 8B | `llamagate/llama-3.1-8b` |
| Llama 3.2 3B | `llamagate/llama-3.2-3b` |
| Mistral 7B v0.3 | `llamagate/mistral-7b-v0.3` |
| Qwen 3 8B | `llamagate/qwen3-8b` |
| Dolphin 3 8B | `llamagate/dolphin3-8b` |
### Reasoning Models
| Model | Model ID |
|-------|----------|
| DeepSeek R1 8B | `llamagate/deepseek-r1-8b` |
| DeepSeek R1 Distill Qwen 7B | `llamagate/deepseek-r1-7b-qwen` |
| OpenThinker 7B | `llamagate/openthinker-7b` |
### Code Models
| Model | Model ID |
|-------|----------|
| Qwen 2.5 Coder 7B | `llamagate/qwen2.5-coder-7b` |
| DeepSeek Coder 6.7B | `llamagate/deepseek-coder-6.7b` |
| CodeLlama 7B | `llamagate/codellama-7b` |
| CodeGemma 7B | `llamagate/codegemma-7b` |
| StarCoder2 7B | `llamagate/starcoder2-7b` |
### Vision Models
| Model | Model ID |
|-------|----------|
| Qwen 3 VL 8B | `llamagate/qwen3-vl-8b` |
| LLaVA 1.5 7B | `llamagate/llava-7b` |
| Gemma 3 4B | `llamagate/gemma3-4b` |
| olmOCR 7B | `llamagate/olmocr-7b` |
| UI-TARS 1.5 7B | `llamagate/ui-tars-7b` |
### Embedding Models
| Model | Model ID |
|-------|----------|
| Nomic Embed Text | `llamagate/nomic-embed-text` |
| Qwen 3 Embedding 8B | `llamagate/qwen3-embedding-8b` |
| EmbeddingGemma 300M | `llamagate/embeddinggemma-300m` |
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="LlamaGate Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# LlamaGate call
response = completion(
model="llamagate/llama-3.1-8b",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="LlamaGate Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# LlamaGate call with streaming
response = completion(
model="llamagate/llama-3.1-8b",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Vision
```python showLineNumbers title="LlamaGate Vision Completion"
import os
import litellm
from litellm import completion
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]
}
]
# LlamaGate vision call
response = completion(
model="llamagate/qwen3-vl-8b",
messages=messages
)
print(response)
```
### Embeddings
```python showLineNumbers title="LlamaGate Embeddings"
import os
import litellm
from litellm import embedding
os.environ["LLAMAGATE_API_KEY"] = "" # your LlamaGate API key
# LlamaGate embedding call
response = embedding(
model="llamagate/nomic-embed-text",
input=["Hello world", "How are you?"]
)
print(response)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export LLAMAGATE_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: llama-3.1-8b
litellm_params:
model: llamagate/llama-3.1-8b
api_key: os.environ/LLAMAGATE_API_KEY
- model_name: deepseek-r1
litellm_params:
model: llamagate/deepseek-r1-8b
api_key: os.environ/LLAMAGATE_API_KEY
- model_name: qwen-coder
litellm_params:
model: llamagate/qwen2.5-coder-7b
api_key: os.environ/LLAMAGATE_API_KEY
```
## Supported OpenAI Parameters
LlamaGate supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature (0-2) |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. JSON mode or JSON schema |
## Pricing
LlamaGate offers competitive per-token pricing:
| Model Category | Input (per 1M) | Output (per 1M) |
|----------------|----------------|-----------------|
| Embeddings | $0.02 | - |
| Small (3-4B) | $0.03-$0.04 | $0.08 |
| Medium (7-8B) | $0.03-$0.15 | $0.05-$0.55 |
| Code Models | $0.06-$0.10 | $0.12-$0.20 |
| Reasoning | $0.08-$0.10 | $0.15-$0.20 |
## Additional Resources
- [LlamaGate Documentation](https://llamagate.dev/docs)
- [LlamaGate Pricing](https://llamagate.dev/pricing)
- [LlamaGate API Reference](https://llamagate.dev/docs/api)
docs/my-website/docs/providers/manus.md
+369
View File
@@ -0,0 +1,369 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Manus
Use Manus AI agents through LiteLLM's OpenAI-compatible Responses API.
| Property | Details |
|----------|---------|
| Description | Manus is an AI agent platform for complex reasoning tasks, document analysis, and multi-step workflows with asynchronous task execution. |
| Provider Route on LiteLLM | `manus/{agent_profile}` |
| Supported Operations | `/responses` (Responses API), `/files` (Files API) |
| Provider Doc | [Manus API ↗](https://open.manus.im/docs/openai-compatibility) |
## Model Format
```shell
manus/{agent_profile}
```
**Examples:**
- `manus/manus-1.6` - General purpose agent
- `manus/manus-1.6-lite` - Lightweight agent for simple tasks
- `manus/manus-1.6-max` - Advanced agent for complex analysis
## LiteLLM Python SDK
```python showLineNumbers title="Basic Usage"
import litellm
import os
import time
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Create task
response = litellm.responses(
model="manus/manus-1.6",
input="What's the capital of France?",
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = litellm.get_response(
response_id=task_id,
custom_llm_provider="manus",
)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
## LiteLLM AI Gateway
### Setup
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: manus-agent
litellm_params:
model: manus/manus-1.6
api_key: os.environ/MANUS_API_KEY
```
```bash title="Start Proxy"
litellm --config config.yaml
```
### Usage
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Create Task"
# Create task
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": "What is the capital of France?"
}'
# Response
{
"id": "task_abc123",
"status": "running",
"metadata": {
"task_url": "https://manus.im/app/task_abc123"
}
}
```
```bash showLineNumbers title="Poll for Completion"
# Check status (repeat until status is "completed")
curl http://localhost:4000/responses/task_abc123 \
-H "Authorization: Bearer your-proxy-key"
# When completed
{
"id": "task_abc123",
"status": "completed",
"output": [
{
"role": "user",
"content": [{"text": "What is the capital of France?"}]
},
{
"role": "assistant",
"content": [{"text": "The capital of France is Paris."}]
}
]
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Create Task and Poll"
import openai
import time
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Create task
response = client.responses.create(
model="manus-agent",
input="What is the capital of France?"
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = client.responses.retrieve(response_id=task_id)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
</TabItem>
</Tabs>
## How It Works
Manus operates as an **asynchronous agent API**:
1. **Create Task**: When you call `litellm.responses()`, Manus creates a task and returns immediately with `status: "running"`
2. **Task Executes**: The agent works on your request in the background
3. **Poll for Completion**: You must repeatedly call `litellm.get_response()` or `client.responses.retrieve()` until the status changes to `"completed"`
4. **Get Results**: Once completed, the `output` field contains the full conversation
**Task Statuses:**
- `running` - Agent is actively working
- `pending` - Agent is waiting for input
- `completed` - Task finished successfully
- `error` - Task failed
:::tip Production Usage
For production applications, use [webhooks](https://open.manus.im/docs/webhooks) instead of polling to get notified when tasks complete.
:::
## Supported Parameters
| Parameter | Supported | Notes |
|-----------|-----------|-------|
| `input` | ✅ | Text, images, or structured content |
| `stream` | ✅ | Fake streaming (task runs async) |
| `max_output_tokens` | ✅ | Limits response length |
| `previous_response_id` | ✅ | For multi-turn conversations |
## Files API
Manus supports file uploads for document analysis and processing. Files can be uploaded and then referenced in Responses API calls.
### LiteLLM Python SDK
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import litellm
import os
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Upload file
file_content = b"This is a document for analysis."
created_file = await litellm.acreate_file(
file=("document.txt", file_content),
purpose="assistants",
custom_llm_provider="manus",
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = await litellm.aresponses(
model="manus/manus-1.6",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id},
],
},
],
extra_body={"task_mode": "agent", "agent_profile": "manus-1.6-agent"},
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = await litellm.afile_retrieve(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"File details: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = await litellm.afile_delete(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"Deleted: {deleted_file.deleted}")
```
### LiteLLM AI Gateway
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Upload File"
# Upload file
curl -X POST http://localhost:4000/v1/files \
-H "Authorization: Bearer your-proxy-key" \
-F "file=@document.txt" \
-F "purpose=assistants" \
-F "custom_llm_provider=manus"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Use File with Responses API"
# Create response with file
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": [
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": "file_abc123"}
]
}
]
}'
```
```bash showLineNumbers title="Retrieve File"
# Get file details
curl http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Delete File"
# Delete file
curl -X DELETE http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"deleted": true
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import openai
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Upload file
with open("document.txt", "rb") as f:
created_file = client.files.create(
file=f,
purpose="assistants",
extra_body={"custom_llm_provider": "manus"}
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = client.responses.create(
model="manus-agent",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id}
]
}
]
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = client.files.retrieve(created_file.id)
print(f"File: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = client.files.delete(created_file.id)
print(f"Deleted: {deleted_file.deleted}")
```
</TabItem>
</Tabs>
## Related Documentation
- [LiteLLM Responses API](/docs/response_api)
- [LiteLLM Files API](/docs/proxy/litellm_managed_files)
- [Manus OpenAI Compatibility](https://open.manus.im/docs/openai-compatibility)
docs/my-website/docs/providers/minimax.md
+639
View File
@@ -0,0 +1,639 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MiniMax
# MiniMax - v1/messages
## Overview
Litellm provides anthropic specs compatible support for minmax
## Supported Models
MiniMax offers three models through their Anthropic-compatible API:
| Model | Description | Input Cost | Output Cost | Prompt Caching Read | Prompt Caching Write |
|-------|-------------|------------|-------------|---------------------|----------------------|
| **MiniMax-M2.1** | Powerful Multi-Language Programming with Enhanced Programming Experience (~60 tps) | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens |
| **MiniMax-M2.1-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | $0.03/M tokens | $0.375/M tokens |
| **MiniMax-M2** | Agentic capabilities, Advanced reasoning | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens |
## Usage Examples
### Basic Chat Completion
```python
import litellm
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/anthropic/v1/messages",
max_tokens=1000
)
print(response.choices[0].message.content)
```
### Using Environment Variables
```bash
export MINIMAX_API_KEY="your-minimax-api-key"
export MINIMAX_API_BASE="https://api.minimax.io/anthropic/v1/messages"
```
```python
import litellm
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}],
max_tokens=1000
)
```
### With Thinking (M2.1 Feature)
```python
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Solve: 2+2=?"}],
thinking={"type": "enabled", "budget_tokens": 1000},
api_key="your-minimax-api-key"
)
# Access thinking content
for block in response.choices[0].message.content:
if hasattr(block, 'type') and block.type == 'thinking':
print(f"Thinking: {block.thinking}")
```
### With Tool Calling
```python
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "What's the weather in SF?"}],
tools=tools,
api_key="your-minimax-api-key",
max_tokens=1000
)
```
## Usage with LiteLLM Proxy
You can use MiniMax models with the Anthropic SDK by routing through LiteLLM Proxy:
| Step | Description |
|------|-------------|
| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` |
| **2. Set Environment Variables** | Point Anthropic SDK to proxy endpoint |
| **3. Use Anthropic SDK** | Call MiniMax models using native Anthropic SDK |
### Step 1: Configure LiteLLM Proxy
Create a `config.yaml`:
```yaml
model_list:
- model_name: minimax/MiniMax-M2.1
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/anthropic/v1/messages
```
Start the proxy:
```bash
litellm --config config.yaml
```
### Step 2: Use with Anthropic SDK
```python
import os
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
import anthropic
client = anthropic.Anthropic()
message = client.messages.create(
model="minimax/MiniMax-M2.1",
max_tokens=1000,
system="You are a helpful assistant.",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hi, how are you?"
}
]
}
]
)
for block in message.content:
if block.type == "thinking":
print(f"Thinking:\n{block.thinking}\n")
elif block.type == "text":
print(f"Text:\n{block.text}\n")
```
# MiniMax - v1/chat/completions
## Usage with LiteLLM SDK
You can use MiniMax's OpenAI-compatible API directly with LiteLLM:
### Basic Chat Completion
```python
import litellm
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello, how are you?"}
],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
print(response.choices[0].message.content)
```
### Using Environment Variables
```bash
export MINIMAX_API_KEY="your-minimax-api-key"
export MINIMAX_API_BASE="https://api.minimax.io/v1"
```
```python
import litellm
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### With Reasoning Split
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Solve: 2+2=?"}
],
extra_body={"reasoning_split": True},
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
# Access reasoning details if available
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking: {response.choices[0].message.reasoning_details}")
print(f"Response: {response.choices[0].message.content}")
```
### With Tool Calling
```python
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "What's the weather in SF?"}],
tools=tools,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
```
### Streaming
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Tell me a story"}],
stream=True,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## Usage with OpenAI SDK via LiteLLM Proxy
You can also use MiniMax models with the OpenAI SDK by routing through LiteLLM Proxy:
| Step | Description |
|------|-------------|
| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` |
| **2. Set Environment Variables** | Point OpenAI SDK to proxy endpoint |
| **3. Use OpenAI SDK** | Call MiniMax models using native OpenAI SDK |
### Step 1: Configure LiteLLM Proxy
Create a `config.yaml`:
```yaml
model_list:
- model_name: minimax/MiniMax-M2.1
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/v1
```
Start the proxy:
```bash
litellm --config config.yaml
```
### Step 2: Use with OpenAI SDK
```python
import os
os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
from openai import OpenAI
client = OpenAI()
response = client.chat.completions.create(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi, how are you?"},
],
# Set reasoning_split=True to separate thinking content
extra_body={"reasoning_split": True},
)
# Access thinking and response
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
print(f"Text:\n{response.choices[0].message.content}\n")
```
### Streaming with OpenAI SDK
```python
from openai import OpenAI
client = OpenAI()
stream = client.chat.completions.create(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Tell me a story"},
],
extra_body={"reasoning_split": True},
stream=True,
)
reasoning_buffer = ""
text_buffer = ""
for chunk in stream:
if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
for detail in chunk.choices[0].delta.reasoning_details:
if "text" in detail:
reasoning_text = detail["text"]
new_reasoning = reasoning_text[len(reasoning_buffer):]
if new_reasoning:
print(new_reasoning, end="", flush=True)
reasoning_buffer = reasoning_text
if chunk.choices[0].delta.content:
content_text = chunk.choices[0].delta.content
new_text = content_text[len(text_buffer):] if text_buffer else content_text
if new_text:
print(new_text, end="", flush=True)
text_buffer = content_text
```
## Cost Calculation
Cost calculation works automatically using the pricing information in `model_prices_and_context_window.json`.
Example:
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}],
api_key="your-minimax-api-key"
)
# Access cost information
print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
```
# MiniMax - Text-to-Speech
## Quick Start
## **LiteLLM Python SDK Usage**
### Basic Usage
```python
from pathlib import Path
from litellm import speech
import os
os.environ["MINIMAX_API_KEY"] = "your-api-key"
speech_file_path = Path(__file__).parent / "speech.mp3"
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="The quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
```
### Async Usage
```python
from litellm import aspeech
from pathlib import Path
import os, asyncio
os.environ["MINIMAX_API_KEY"] = "your-api-key"
async def test_async_speech():
speech_file_path = Path(__file__).parent / "speech.mp3"
response = await aspeech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="The quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
asyncio.run(test_async_speech())
```
### Voice Selection
MiniMax supports many voices. LiteLLM provides OpenAI-compatible voice names that map to MiniMax voices:
```python
from litellm import speech
# OpenAI-compatible voice names
voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
for voice in voices:
response = speech(
model="minimax/speech-2.6-hd",
voice=voice,
input=f"This is the {voice} voice",
)
response.stream_to_file(f"speech_{voice}.mp3")
```
You can also use MiniMax-native voice IDs directly:
```python
response = speech(
model="minimax/speech-2.6-hd",
voice="male-qn-qingse", # MiniMax native voice ID
input="Using native MiniMax voice ID",
)
```
### Custom Parameters
MiniMax TTS supports additional parameters for fine-tuning audio output:
```python
from litellm import speech
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="Custom audio parameters",
speed=1.5, # Speed: 0.5 to 2.0
response_format="mp3", # Format: mp3, pcm, wav, flac
extra_body={
"vol": 1.2, # Volume: 0.1 to 10
"pitch": 2, # Pitch adjustment: -12 to 12
"sample_rate": 32000, # 16000, 24000, or 32000
"bitrate": 128000, # For MP3: 64000, 128000, 192000, 256000
"channel": 1, # 1 for mono, 2 for stereo
}
)
response.stream_to_file("custom_speech.mp3")
```
### Response Formats
```python
from litellm import speech
# MP3 format (default)
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="MP3 format audio",
response_format="mp3",
)
# PCM format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="PCM format audio",
response_format="pcm",
)
# WAV format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="WAV format audio",
response_format="wav",
)
# FLAC format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="FLAC format audio",
response_format="flac",
)
```
## **LiteLLM Proxy Usage**
LiteLLM provides an OpenAI-compatible `/audio/speech` endpoint for MiniMax TTS.
### Setup
Add MiniMax to your proxy configuration:
```yaml
model_list:
- model_name: tts
litellm_params:
model: minimax/speech-2.6-hd
api_key: os.environ/MINIMAX_API_KEY
- model_name: tts-turbo
litellm_params:
model: minimax/speech-2.6-turbo
api_key: os.environ/MINIMAX_API_KEY
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### Making Requests
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "tts",
"input": "The quick brown fox jumped over the lazy dog.",
"voice": "alloy"
}' \
--output speech.mp3
```
With custom parameters:
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "tts",
"input": "Custom parameters example.",
"voice": "nova",
"speed": 1.5,
"response_format": "mp3",
"extra_body": {
"vol": 1.2,
"pitch": 1,
"sample_rate": 32000
}
}' \
--output custom_speech.mp3
```
## Voice Mappings
LiteLLM maps OpenAI-compatible voice names to MiniMax voice IDs:
| OpenAI Voice | MiniMax Voice ID | Description |
|--------------|------------------|-------------|
| alloy | male-qn-qingse | Male voice |
| echo | male-qn-jingying | Male voice |
| fable | female-shaonv | Female voice |
| onyx | male-qn-badao | Male voice |
| nova | female-yujie | Female voice |
| shimmer | female-tianmei | Female voice |
You can also use any MiniMax-native voice ID directly by passing it as the `voice` parameter.
### Streaming (WebSocket)
:::note
The current implementation uses MiniMax's HTTP endpoint. For WebSocket streaming support, please refer to MiniMax's official documentation at [https://platform.minimax.io/docs](https://platform.minimax.io/docs).
:::
## Error Handling
```python
from litellm import speech
import litellm
try:
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="Test input",
)
response.stream_to_file("output.mp3")
except litellm.exceptions.BadRequestError as e:
print(f"Bad request: {e}")
except litellm.exceptions.AuthenticationError as e:
print(f"Authentication failed: {e}")
except Exception as e:
print(f"Error: {e}")
```
### Extra Body Parameters
Pass these via `extra_body`:
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| vol | float | Volume (0.1 to 10) | 1.0 |
| pitch | int | Pitch adjustment (-12 to 12) | 0 |
| sample_rate | int | Sample rate: 16000, 24000, 32000 | 32000 |
| bitrate | int | Bitrate for MP3: 64000, 128000, 192000, 256000 | 128000 |
| channel | int | Audio channels: 1 (mono) or 2 (stereo) | 1 |
| output_format | string | Output format: "hex" or "url" (url returns a URL valid for 24 hours) | hex |
docs/my-website/docs/providers/nano-gpt.md
+170
View File
@@ -0,0 +1,170 @@
# NanoGPT
## Overview
| Property | Details |
|-------|-------|
| Description | NanoGPT is a pay-per-prompt and subscription based AI service providing instant access to over 200+ powerful AI models with no subscriptions or registration required. |
| Provider Route on LiteLLM | `nano-gpt/` |
| Link to Provider Doc | [NanoGPT Website ↗](https://nano-gpt.com) |
| Base URL | `https://nano-gpt.com/api/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage), [`/completions`](#text-completion), [`/embeddings`](#embeddings) |
<br />
## What is NanoGPT?
NanoGPT is a flexible AI API service that offers:
- **Pay-Per-Prompt Pricing**: No subscriptions, pay only for what you use
- **200+ AI Models**: Access to text, image, and video generation models
- **No Registration Required**: Get started instantly
- **OpenAI-Compatible API**: Easy integration with existing code
- **Streaming Support**: Real-time response streaming
- **Tool Calling**: Support for function calling
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
```
Get your NanoGPT API key from [nano-gpt.com](https://nano-gpt.com).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="NanoGPT Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# NanoGPT call
response = completion(
model="nano-gpt/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="NanoGPT Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# NanoGPT call with streaming
response = completion(
model="nano-gpt/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Tool Calling
```python showLineNumbers title="NanoGPT Tool Calling"
import os
import litellm
os.environ["NANOGPT_API_KEY"] = ""
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
}
}
}
}
]
response = litellm.completion(
model="nano-gpt/model-name",
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
tools=tools
)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export NANOGPT_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: nano-gpt-model
litellm_params:
model: nano-gpt/model-name # Replace with actual model name
api_key: os.environ/NANOGPT_API_KEY
```
## Supported OpenAI Parameters
NanoGPT supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 200+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `n` | integer | Optional. Number of completions to generate |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
| `user` | string | Optional. User identifier |
## Model Categories
NanoGPT provides access to multiple model categories:
- **Text Generation**: 200+ LLMs for chat, completion, and analysis
- **Image Generation**: AI models for creating images
- **Video Generation**: AI models for video creation
- **Embedding Models**: Text embedding models for vector search
## Pricing Model
NanoGPT offers a flexible pricing structure:
- **Pay-Per-Prompt**: No subscription required
- **No Registration**: Get started immediately
- **Transparent Pricing**: Pay only for what you use
## API Documentation
For detailed API documentation, visit [docs.nano-gpt.com](https://docs.nano-gpt.com).
## Additional Resources
- [NanoGPT Website](https://nano-gpt.com)
- [NanoGPT API Documentation](https://nano-gpt.com/api)
- [NanoGPT Model List](https://docs.nano-gpt.com/api-reference/endpoint/models)
docs/my-website/docs/providers/openai.md
+1 -1
View File
@@ -495,7 +495,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|-------|----------------------|------------------|
| `gpt-5.1` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
docs/my-website/docs/providers/openrouter.md
+133 -1
View File
@@ -1,5 +1,5 @@
# OpenRouter
LiteLLM supports all the text / chat / vision models from [OpenRouter](https://openrouter.ai/docs)
LiteLLM supports all the text / chat / vision / embedding models from [OpenRouter](https://openrouter.ai/docs)
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_OpenRouter.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
@@ -78,3 +78,135 @@ response = completion(
route= ""
)
```
## Embedding
```python
from litellm import embedding
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = embedding(
model="openrouter/openai/text-embedding-3-small",
input=["good morning from litellm", "this is another item"],
)
print(response)
```
## Image Generation
OpenRouter supports image generation through select models like Google Gemini image generation models. LiteLLM transforms standard image generation requests to OpenRouter's chat completion format.
### Supported Parameters
- `size`: Maps to OpenRouter's `aspect_ratio` format
- `1024x1024` → `1:1` (square)
- `1536x1024` → `3:2` (landscape)
- `1024x1536` → `2:3` (portrait)
- `1792x1024` → `16:9` (wide landscape)
- `1024x1792` → `9:16` (tall portrait)
- `quality`: Maps to OpenRouter's `image_size` format (Gemini models)
- `low` or `standard` → `1K`
- `medium` → `2K`
- `high` or `hd` → `4K`
- `n`: Number of images to generate
### Usage
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Basic image generation
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
)
print(response)
```
### Advanced Usage with Parameters
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Generate high-quality landscape image
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A serene mountain landscape with a lake",
size="1536x1024", # Landscape format
quality="high", # High quality (4K)
)
# Access the generated image
image_data = response.data[0]
if image_data.b64_json:
# Base64 encoded image
print(f"Generated base64 image: {image_data.b64_json[:50]}...")
elif image_data.url:
# Image URL
print(f"Generated image URL: {image_data.url}")
```
### Using OpenRouter-Specific Parameters
You can also pass OpenRouter-specific parameters directly using `image_config`:
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A futuristic cityscape at night",
image_config={
"aspect_ratio": "16:9", # OpenRouter native format
"image_size": "4K" # OpenRouter native format
}
)
print(response)
```
### Response Format
The response follows the standard LiteLLM ImageResponse format:
```python
{
"created": 1703658209,
"data": [{
"b64_json": "iVBORw0KGgoAAAANSUhEUgAA...", # Base64 encoded image
"url": None,
"revised_prompt": None
}],
"usage": {
"input_tokens": 10,
"output_tokens": 1290,
"total_tokens": 1300
}
}
```
### Cost Tracking
OpenRouter provides cost information in the response, which LiteLLM automatically tracks:
```python
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A cute baby sea otter",
)
# Cost is available in the response metadata
print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
```
docs/my-website/docs/providers/poe.md
+139
View File
@@ -0,0 +1,139 @@
# Poe
## Overview
| Property | Details |
|-------|-------|
| Description | Poe is Quora's AI platform that provides access to more than 100 models across text, image, video, and voice modalities through a developer-friendly API. |
| Provider Route on LiteLLM | `poe/` |
| Link to Provider Doc | [Poe Website ↗](https://poe.com) |
| Base URL | `https://api.poe.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Poe?
Poe is Quora's comprehensive AI platform that offers:
- **100+ Models**: Access to a wide variety of AI models
- **Multiple Modalities**: Text, image, video, and voice AI
- **Popular Models**: Including OpenAI's GPT series and Anthropic's Claude
- **Developer API**: Easy integration for applications
- **Extensive Reach**: Benefits from Quora's 400M monthly unique visitors
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["POE_API_KEY"] = "" # your Poe API key
```
Get your Poe API key from the [Poe platform](https://poe.com).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Poe Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["POE_API_KEY"] = "" # your Poe API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Poe call
response = completion(
model="poe/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Poe Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["POE_API_KEY"] = "" # your Poe API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Poe call with streaming
response = completion(
model="poe/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export POE_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: poe-model
litellm_params:
model: poe/model-name # Replace with actual model name
api_key: os.environ/POE_API_KEY
```
## Supported OpenAI Parameters
Poe supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 100+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
| `user` | string | Optional. User identifier |
## Available Model Categories
Poe provides access to models across multiple providers:
- **OpenAI Models**: Including GPT-4, GPT-4 Turbo, GPT-3.5 Turbo
- **Anthropic Models**: Including Claude 3 Opus, Sonnet, Haiku
- **Other Popular Models**: Various provider models available
- **Multi-Modal**: Text, image, video, and voice models
## Platform Benefits
Using Poe through LiteLLM offers several advantages:
- **Unified Access**: Single API for many different models
- **Quora Integration**: Access to large user base and content ecosystem
- **Content Sharing**: Capabilities to share model outputs with followers
- **Content Distribution**: Best AI content distributed to all users
- **Model Discovery**: Efficient way to explore new AI models
## Developer Resources
Poe is actively building developer features and welcomes early access requests for API integration.
## Additional Resources
- [Poe Website](https://poe.com)
- [Poe AI Quora Space](https://poeai.quora.com)
- [Quora Blog Post about Poe](https://quorablog.quora.com/Poe)
docs/my-website/docs/providers/synthetic.md
+119
View File
@@ -0,0 +1,119 @@
# Synthetic
## Overview
| Property | Details |
|-------|-------|
| Description | Synthetic runs open-source AI models in secure datacenters within the US and EU, with a focus on privacy. They never train on your data and auto-delete API data within 14 days. |
| Provider Route on LiteLLM | `synthetic/` |
| Link to Provider Doc | [Synthetic Website ↗](https://synthetic.new) |
| Base URL | `https://api.synthetic.new/openai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Synthetic?
Synthetic is a privacy-focused AI platform that provides access to open-source LLMs with the following guarantees:
- **Privacy-First**: Data never used for training
- **Secure Hosting**: Models run in secure datacenters in US and EU
- **Auto-Deletion**: API data automatically deleted within 14 days
- **Open Source**: Runs open-source AI models
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
```
Get your Synthetic API key from [synthetic.new](https://synthetic.new).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Synthetic Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Synthetic call
response = completion(
model="synthetic/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Synthetic Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Synthetic call with streaming
response = completion(
model="synthetic/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export SYNTHETIC_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: synthetic-model
litellm_params:
model: synthetic/model-name # Replace with actual model name
api_key: os.environ/SYNTHETIC_API_KEY
```
## Supported OpenAI Parameters
Synthetic supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
## Privacy & Security
Synthetic provides enterprise-grade privacy protections:
- Data auto-deleted within 14 days
- No data used for model training
- Secure hosting in US and EU datacenters
- Compliance-friendly architecture
## Additional Resources
- [Synthetic Website](https://synthetic.new)
docs/my-website/docs/providers/xiaomi_mimo.md
+137
View File
@@ -0,0 +1,137 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Xiaomi MiMo
https://platform.xiaomimimo.com/#/docs
:::tip
**We support ALL Xiaomi MiMo models, just set `model=xiaomi_mimo/<any-model-on-xiaomi-mimo>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
os.environ['XIAOMI_MIMO_API_KEY']
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['XIAOMI_MIMO_API_KEY'] = ""
response = completion(
model="xiaomi_mimo/mimo-v2-flash",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
max_tokens=1024,
temperature=0.3,
top_p=0.95,
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['XIAOMI_MIMO_API_KEY'] = ""
response = completion(
model="xiaomi_mimo/mimo-v2-flash",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
stream=True,
max_tokens=1024,
temperature=0.3,
top_p=0.95,
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy Server
Here's how to call a Xiaomi MiMo model with the LiteLLM Proxy Server
1. Modify the config.yaml
```yaml
model_list:
- model_name: my-model
litellm_params:
model: xiaomi_mimo/<your-model-name> # add xiaomi_mimo/ prefix to route as Xiaomi MiMo provider
api_key: api-key # api key to send your model
```
2. Start the proxy
```bash
$ litellm --config /path/to/config.yaml
```
3. Send Request to LiteLLM Proxy Server
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="my-model",
messages = [
{
"role": "user",
"content": "what llm are you"
}
],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "my-model",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
## Supported Models
| Model Name | Usage |
|------------|-------|
| mimo-v2-flash | `completion(model="xiaomi_mimo/mimo-v2-flash", messages)` |
docs/my-website/docs/providers/zai.md
+19 -17
View File
@@ -19,7 +19,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@@ -34,7 +34,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@@ -51,7 +51,8 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
| Model Name | Function Call | Notes |
|------------|---------------|-------|
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
| glm-4.7 | `completion(model="zai/glm-4.7", messages)` | **Latest flagship**, 200K context, **Reasoning** |
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | 200K context |
| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
@@ -62,16 +63,17 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
## Model Pricing
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|----------------|
| glm-4.6 | $0.60 | $2.20 | 200K |
| glm-4.5 | $0.60 | $2.20 | 128K |
| glm-4.5v | $0.60 | $1.80 | 128K |
| glm-4.5-x | $2.20 | $8.90 | 128K |
| glm-4.5-air | $0.20 | $1.10 | 128K |
| glm-4.5-airx | $1.10 | $4.50 | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
| glm-4.5-flash | **FREE** | **FREE** | 128K |
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Cached Input ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|---------------------------|----------------|
| glm-4.7 | $0.60 | $2.20 | $0.11 | 200K |
| glm-4.6 | $0.60 | $2.20 | - | 200K |
| glm-4.5 | $0.60 | $2.20 | - | 128K |
| glm-4.5v | $0.60 | $1.80 | - | 128K |
| glm-4.5-x | $2.20 | $8.90 | - | 128K |
| glm-4.5-air | $0.20 | $1.10 | - | 128K |
| glm-4.5-airx | $1.10 | $4.50 | - | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | - | 128K |
| glm-4.5-flash | **FREE** | **FREE** | - | 128K |
## Using with LiteLLM Proxy
@@ -84,7 +86,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
@@ -98,9 +100,9 @@ print(response.choices[0].message.content)
```yaml
model_list:
- model_name: glm-4.6
- model_name: glm-4.7
litellm_params:
model: zai/glm-4.6
model: zai/glm-4.7
api_key: os.environ/ZAI_API_KEY
- model_name: glm-4.5-flash # Free tier
litellm_params:
@@ -121,7 +123,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "glm-4.6",
"model": "glm-4.7",
"messages": [
{
"role": "user",
docs/my-website/docs/proxy/access_control.md
+1 -1
View File
@@ -51,7 +51,7 @@ LiteLLM has two types of roles:
| Role Name | Permissions |
|-----------|-------------|
| `org_admin` | Admin over a specific organization. Can create teams and users within their organization ✨ **Premium Feature** |
| `team_admin` | Admin over a specific team. Can manage team members, update team settings, and create keys for their team. ✨ **Premium Feature** |
| `team_admin` | Admin over a specific team. Can manage team members, update team member permissions, and create keys for their team. ✨ **Premium Feature** |
## What Can Each Role Do?
docs/my-website/docs/proxy/admin_ui_sso.md
+50 -1
View File
@@ -73,8 +73,21 @@ GOOGLE_CLIENT_SECRET=
```shell
MICROSOFT_CLIENT_ID="84583a4d-"
MICROSOFT_CLIENT_SECRET="nbk8Q~"
MICROSOFT_TENANT="5a39737
MICROSOFT_TENANT="5a39737"
```
**Optional: Custom Microsoft SSO Endpoints**
If you need to use custom Microsoft SSO endpoints (e.g., for a custom identity provider, sovereign cloud, or proxy), you can override the default endpoints:
```shell
MICROSOFT_AUTHORIZATION_ENDPOINT="https://your-custom-url.com/oauth2/v2.0/authorize"
MICROSOFT_TOKEN_ENDPOINT="https://your-custom-url.com/oauth2/v2.0/token"
MICROSOFT_USERINFO_ENDPOINT="https://your-custom-graph-api.com/v1.0/me"
```
If these are not set, the default Microsoft endpoints are used based on your tenant.
- Set Redirect URI on your App Registration on https://portal.azure.com/
- Set a redirect url = `<your proxy base url>/sso/callback`
```shell
@@ -98,6 +111,42 @@ To set up app roles:
4. Assign users to these roles in your Enterprise Application
5. When users sign in via SSO, LiteLLM will automatically assign them the corresponding role
**Advanced: Custom User Attribute Mapping**
For certain Microsoft Entra ID configurations, you may need to override the default user attribute field names. This is useful when your organization uses custom claims or non-standard attribute names in the SSO response.
**Step 1: Debug SSO Response**
First, inspect the JWT fields returned by your Microsoft SSO provider using the [SSO Debug Route](#debugging-sso-jwt-fields).
1. Add `/sso/debug/callback` as a redirect URL in your Azure App Registration
2. Navigate to `https://<proxy_base_url>/sso/debug/login`
3. Complete the SSO flow to see the returned user attributes
**Step 2: Identify Field Attribute Names**
From the debug response, identify the field names used for email, display name, user ID, first name, and last name.
**Step 3: Set Environment Variables**
Override the default attribute names by setting these environment variables:
| Environment Variable | Description | Default Value |
|---------------------|-------------|---------------|
| `MICROSOFT_USER_EMAIL_ATTRIBUTE` | Field name for user email | `userPrincipalName` |
| `MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE` | Field name for display name | `displayName` |
| `MICROSOFT_USER_ID_ATTRIBUTE` | Field name for user ID | `id` |
| `MICROSOFT_USER_FIRST_NAME_ATTRIBUTE` | Field name for first name | `givenName` |
| `MICROSOFT_USER_LAST_NAME_ATTRIBUTE` | Field name for last name | `surname` |
**Step 4: Restart the Proxy**
After setting the environment variables, restart the proxy:
```bash
litellm --config /path/to/config.yaml
```
</TabItem>
<TabItem value="Generic" label="Generic SSO Provider">

Some files were not shown because too many files have changed in this diff Show More