tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-07 20:22:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'litellm_staging_01_19_2026' into fix/bedrock-thinking-tool-call-2

Browse Source
This commit is contained in:
Benedikt Óskarsson
2026-01-19 15:18:47 +00:00
committed by GitHub
parent 0367e9c9f1 fc9988b686
commit 406cdbe321
626 changed files with 33819 additions and 5441 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+1 -39
View File
@@ -2036,6 +2036,7 @@ jobs:
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
- run: python ./tests/code_coverage_tests/test_proxy_types_import.py
- run: python ./tests/code_coverage_tests/callback_manager_test.py
- run: python ./tests/code_coverage_tests/recursive_detector.py
@@ -2054,39 +2055,6 @@ jobs:
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
memory_leak_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- setup_litellm_test_deps
- run:
name: Install Memory Test Dependencies
command: |
pip install "psutil>=5.9.0"
pip install "fastapi>=0.100.0"
pip install "httpx>=0.24.0"
pip install "uvicorn>=0.23.0"
- run:
name: Run Linear Memory Growth Tests
command: |
echo "Running memory leak tests individually to avoid baseline drift..."
echo "Running test_memory_baseline_1k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_1k -v -s --tb=short
echo "Running test_memory_baseline_2k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_2k -v -s --tb=short
echo "Running test_memory_baseline_4k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_4k -v -s --tb=short
echo "Running test_memory_baseline_10k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_10k -v -s --tb=short
echo "Running test_memory_baseline_30k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_30k -v -s --tb=short
no_output_timeout: 60m
db_migration_disable_update_check:
machine:
image: ubuntu-2204:2023.10.1
@@ -3837,12 +3805,6 @@ workflows:
only:
- main
- /litellm_.*/
- memory_leak_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
.github/ISSUE_TEMPLATE/bug_report.yml
+10
View File
@@ -7,6 +7,16 @@ body:
attributes:
value: |
Thanks for taking the time to fill out this bug report!
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the bug you encountered.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: what-happened
attributes:
.github/ISSUE_TEMPLATE/feature_request.yml
+8
View File
@@ -7,6 +7,14 @@ body:
attributes:
value: |
Thanks for making LiteLLM better!
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the feature you are requesting.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: the-feature
attributes:
.github/workflows/check_duplicate_issues.yml
+29
View File
@@ -0,0 +1,29 @@
name: Check Duplicate Issues
on:
issues:
types: [opened, edited]
jobs:
check-duplicate:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
.github/workflows/label-component.yml
+34
View File
@@ -80,3 +80,37 @@ jobs:
break;
}
}
// Check for 'claude code' keyword (can be applied alongside component labels)
if (/claude code/i.test(body)) {
const claudeLabel = {
name: 'claude code',
color: '7c3aed',
description: 'Issues related to Claude Code usage'
};
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name,
color: claudeLabel.color,
description: claudeLabel.description
});
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [claudeLabel.name]
});
}
.github/workflows/test-litellm.yml
+1
View File
@@ -35,6 +35,7 @@ jobs:
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
.gitignore
+1
View File
@@ -59,6 +59,7 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/_next/
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
ARCHITECTURE.md
+398
View File
@@ -0,0 +1,398 @@
# LiteLLM Architecture - LiteLLM SDK + AI Gateway
This document helps contributors understand where to make changes in LiteLLM.
---
## How It Works
The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
```
OpenAI SDK (client) ──▶ LiteLLM AI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Anthropic SDK (client) ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Any HTTP client ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
```
The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
---
## 1. AI Gateway (Proxy) Request Flow
The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
```mermaid
sequenceDiagram
participant Client
participant ProxyServer as proxy/proxy_server.py
participant Auth as proxy/auth/user_api_key_auth.py
participant Redis as Redis Cache
participant Hooks as proxy/hooks/
participant Router as router.py
participant Main as main.py + utils.py
participant Handler as llms/custom_httpx/llm_http_handler.py
participant Transform as llms/{provider}/chat/transformation.py
participant Provider as LLM Provider API
participant CostCalc as cost_calculator.py
participant LoggingObj as litellm_logging.py
participant DBWriter as db/db_spend_update_writer.py
participant Postgres as PostgreSQL
%% Request Flow
Client->>ProxyServer: POST /v1/chat/completions
ProxyServer->>Auth: user_api_key_auth()
Auth->>Redis: Check API key cache
Redis-->>Auth: Key info + spend limits
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
Hooks->>Redis: Check/increment rate limit counters
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
Main->>Handler: BaseLLMHTTPHandler.completion()
Handler->>Transform: ProviderConfig.transform_request()
Handler->>Provider: HTTP Request
Provider-->>Handler: Response
Handler->>Transform: ProviderConfig.transform_response()
Transform-->>Handler: ModelResponse
Handler-->>Main: ModelResponse
%% Cost Attribution (in utils.py wrapper)
Main->>LoggingObj: update_response_metadata()
LoggingObj->>CostCalc: _response_cost_calculator()
CostCalc->>CostCalc: completion_cost(tokens × price)
CostCalc-->>LoggingObj: response_cost
LoggingObj-->>Main: Set response._hidden_params["response_cost"]
Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
%% Response Headers + Async Logging
ProxyServer->>ProxyServer: Extract cost from hidden_params
ProxyServer->>LoggingObj: async_success_handler()
LoggingObj->>Hooks: async_log_success_event()
Hooks->>DBWriter: update_database(response_cost)
DBWriter->>Redis: Queue spend increment
DBWriter->>Postgres: Batch write spend logs (async)
ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
```
### Proxy Components
```mermaid
graph TD
subgraph "Incoming Request"
Client["POST /v1/chat/completions"]
end
subgraph "proxy/proxy_server.py"
Endpoint["chat_completion()"]
end
subgraph "proxy/auth/"
Auth["user_api_key_auth()"]
end
subgraph "proxy/"
PreCall["litellm_pre_call_utils.py"]
RouteRequest["route_llm_request.py"]
end
subgraph "litellm/"
Router["router.py"]
Main["main.py"]
end
subgraph "Infrastructure"
DualCache["DualCache<br/>(in-memory + Redis)"]
Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
end
Client --> Endpoint
Endpoint --> Auth
Auth --> DualCache
DualCache -.->|cache miss| Postgres
Auth --> PreCall
PreCall --> RouteRequest
RouteRequest --> Router
Router --> DualCache
Router --> Main
Main --> Client
```
**Key proxy files:**
- `proxy/proxy_server.py` - Main API endpoints
- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
- `proxy/hooks/` - Proxy-level callbacks
- `router.py` - Load balancing, fallbacks
- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
**LLM-specific proxy endpoints:**
| Endpoint | Directory | Purpose |
|----------|-----------|---------|
| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
**Proxy Hooks** (`proxy/hooks/__init__.py`):
| Hook | File | Purpose |
|------|------|---------|
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
### Infrastructure Components
The AI Gateway uses external infrastructure for persistence and caching:
```mermaid
graph LR
subgraph "AI Gateway (proxy/)"
Proxy["proxy_server.py"]
Auth["auth/user_api_key_auth.py"]
DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
InternalCache["utils.py<br/>InternalUsageCache"]
CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
Scheduler["APScheduler<br/>ProxyStartupEvent"]
end
subgraph "SDK (litellm/)"
Router["router.py<br/>Router.cache (DualCache)"]
LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
CacheClass["caching/caching.py<br/>Cache"]
end
subgraph "Redis (caching/redis_cache.py)"
RateLimit["Rate Limit Counters"]
SpendQueue["Spend Increment Queue"]
KeyCache["API Key Cache"]
TPM_RPM["TPM/RPM Tracking"]
Cooldowns["Deployment Cooldowns"]
LLMResponseCache["LLM Response Cache"]
end
subgraph "PostgreSQL (proxy/schema.prisma)"
Keys["LiteLLM_VerificationToken"]
Teams["LiteLLM_TeamTable"]
SpendLogs["LiteLLM_SpendLogs"]
Users["LiteLLM_UserTable"]
end
Auth --> InternalCache
InternalCache --> KeyCache
InternalCache -.->|cache miss| Keys
InternalCache --> RateLimit
Router --> TPM_RPM
Router --> Cooldowns
LLMCache --> CacheClass
CacheClass --> LLMResponseCache
CostCallback --> DBWriter
DBWriter --> SpendQueue
DBWriter --> SpendLogs
Scheduler --> SpendLogs
Scheduler --> Keys
```
| Component | Purpose | Key Files/Classes |
|-----------|---------|-------------------|
| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
| Job | Interval | Purpose | Key Files |
|-----|----------|---------|-----------|
| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
**Cost Attribution Flow:**
1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
4. Cost is stored in `response._hidden_params["response_cost"]`
5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
---
## 2. SDK Request Flow
The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
```mermaid
graph TD
subgraph "SDK Entry Points"
Completion["litellm.completion()"]
Messages["litellm.messages()"]
end
subgraph "main.py"
Main["completion()<br/>acompletion()"]
end
subgraph "utils.py"
GetProvider["get_llm_provider()"]
end
subgraph "llms/custom_httpx/"
Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
end
subgraph "llms/{provider}/chat/"
TransformReq["transform_request()"]
TransformResp["transform_response()"]
end
subgraph "litellm_core_utils/"
Streaming["streaming_handler.py"]
end
subgraph "integrations/ (async, off main thread)"
Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
end
Completion --> Main
Messages --> Main
Main --> GetProvider
GetProvider --> Handler
Handler --> TransformReq
TransformReq --> HTTP
HTTP --> Provider["LLM Provider API"]
Provider --> HTTP
HTTP --> TransformResp
TransformResp --> Streaming
Streaming --> Response["ModelResponse"]
Response -.->|async| Callbacks
```
**Key SDK files:**
- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
- `utils.py` - `get_llm_provider()` resolves model → provider
- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
- `litellm_core_utils/streaming_handler.py` - Streaming response handling
- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
---
## 3. Translation Layer
When a request comes in, it goes through a **translation layer** that converts between API formats.
Each translation is isolated in its own file, making it easy to test and modify independently.
### Where to find translations
| Incoming API | Provider | Translation File |
|--------------|----------|------------------|
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
### Example: Debugging prompt caching
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
3. Compare how each handles `cache_control` in `transform_request()`
### How translations work
Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
```python
class ProviderConfig(BaseConfig):
def transform_request(self, model, messages, optional_params, litellm_params, headers):
# Convert OpenAI format → Provider format
return {"messages": transformed_messages, ...}
def transform_response(self, model, raw_response, model_response, logging_obj, ...):
# Convert Provider format → OpenAI format
return ModelResponse(choices=[...], usage=Usage(...))
```
The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
---
## 4. Adding/Modifying Providers
### To add a new provider:
1. Create `llms/{provider}/chat/transformation.py`
2. Implement `Config` class with `transform_request()` and `transform_response()`
3. Add tests in `tests/llm_translation/test_{provider}.py`
### To add a feature (e.g., prompt caching):
1. Find the translation file from the table above
2. Modify `transform_request()` to handle the new parameter
3. Add unit tests that verify the transformation
### Testing checklist
When adding a feature, verify it works across all paths:
| Test | File Pattern |
|------|--------------|
| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
| Vertex AI | `tests/llm_translation/test_vertex*.py` |
| Gemini | `tests/llm_translation/test_gemini*.py` |
### Unit testing translations
Translations are designed to be unit testable without making API calls:
```python
from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
def test_prompt_caching_transform():
config = BedrockConverseConfig()
result = config.transform_request(
model="anthropic.claude-3-opus",
messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
optional_params={},
litellm_params={},
headers={}
)
assert "cachePoint" in str(result) # Verify cache_control was translated
```
Makefile
+2 -1
View File
@@ -45,6 +45,7 @@ install-proxy-dev-ci:
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install openapi-core
cd enterprise && poetry run pip install -e . && cd ..
install-helm-unittest:
@@ -100,4 +101,4 @@ test-llm-translation-single: install-test-deps
@mkdir -p test-results
poetry run pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
-v --tb=short --maxfail=100 --timeout=300
batch_small.jsonl
-4
View File
@@ -1,4 +0,0 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}
ci_cd/.grype.yaml
+2 -2
View File
@@ -1,3 +1,3 @@
ignore:
- vulnerability: CVE-2019-1010022
reason: no fixed glibc package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
ci_cd/security_scans.sh
+3
View File
@@ -129,11 +129,14 @@ run_grype_scans() {
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
)
# Build JSON array of allowlisted CVE IDs for jq
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
+195
View File
@@ -0,0 +1,195 @@
# Claude Code with LiteLLM Quickstart
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
## Video Walkthrough
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Step 1: Setup config.yaml
Create a secure configuration using environment variables:
```yaml
model_list:
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
## Step 2: Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
## Step 3: Verify Setup
Test that your proxy is working correctly:
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
## Step 4: Configure Claude Code
### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
## Step 5: Use Claude Code
Start Claude Code and it will automatically use your configured models:
```bash
# Claude Code will use the models configured in your LiteLLM proxy
claude
# Or specify a model if you have multiple configured
claude --model claude-3-5-sonnet-20241022
claude --model claude-3-5-haiku-20241022
```
## Troubleshooting
Common issues and solutions:
**Claude Code not connecting:**
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` is set correctly
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
- Check LiteLLM logs for detailed error messages
## Using Multiple Models and Providers
Expand your configuration to support multiple providers and models:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Switch between models seamlessly:
```bash
# Use Claude for complex reasoning
claude --model claude-3-5-sonnet-20241022
# Use Haiku for fast responses
claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
```
## Additional Resources
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
cookbook/ai_coding_tool_guides/index.json
+134
View File
@@ -0,0 +1,134 @@
[{
"title": "Claude Code Quickstart",
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
},
{
"title": "Claude Code with MCPs",
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"MCP"
]
},
{
"title": "Claude Code with Non-Anthropic Models",
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"OpenAI",
"Gemini"
]
},
{
"title": "Cursor Quickstart",
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Cursor",
"LiteLLM",
"Quickstart"
]
},
{
"title": "Github Copilot Quickstart",
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Github Copilot",
"LiteLLM",
"Quickstart"
]
},
{
"title": "LiteLLM Gemini CLI Quickstart",
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Gemini CLI",
"Gemini",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenAI Codex CLI Quickstart",
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenAI Codex CLI",
"OpenAI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenWebUI Quickstart",
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenWebUI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "AI Coding Tool Usage Tracking",
"description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"Gemini CLI",
"OpenAI Codex",
"LiteLLM"
]
},
{
"title": "Use Web Search with Claude Code (across OpenAI/Anthropic/Gemini/etc.)",
"description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"Web Search"
]
},
{
"title": "Track Claude Code Usage per user via Custom Headers",
"description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
}]
deploy/charts/litellm-helm/templates/deployment.yaml
+2 -1
View File
@@ -170,7 +170,8 @@ spec:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp
deploy/charts/litellm-helm/tests/deployment_tests.yaml
+2 -1
View File
@@ -136,7 +136,8 @@ tests:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
- it: should work with lifecycle hooks
template: deployment.yaml
set:
docs/my-website/docs/caching/all_caches.md
+8
View File
@@ -105,6 +105,14 @@ Then simply initialize:
litellm.cache = Cache(type="redis")
```
:::info
Use `REDIS_*` environment variables as the primary mechanism for configuring all Redis client library parameters. This approach automatically maps environment variables to Redis client kwargs and is the suggested way to toggle Redis settings.
:::
:::warning
If you need to pass non-string Redis parameters (integers, booleans, complex objects), avoid `REDIS_*` environment variables as they may fail during Redis client initialization. Instead, pass them directly as kwargs to the `Cache()` constructor.
:::
</TabItem>
<TabItem value="gcs" label="gcs-cache">
docs/my-website/docs/image_generation.md
+23 -1
View File
@@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
## Quick Start
@@ -238,6 +238,27 @@ print(response)
See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
## OpenRouter Image Generation Models
Use this for image generation models available through OpenRouter (e.g., Google Gemini image generation models)
#### Usage
```python showLineNumbers
from litellm import image_generation
import os
os.environ['OPENROUTER_API_KEY'] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
size="1024x1024",
quality="high",
)
print(response)
```
## OpenAI Compatible Image Generation Models
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
@@ -301,5 +322,6 @@ print(f"response: {response}")
| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
| OpenRouter | [OpenRouter Image Generation →](./providers/openrouter#image-generation) |
| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) |
docs/my-website/docs/mcp.md
+18 -1
View File
@@ -60,6 +60,8 @@ model_list:
If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
For diagnosing connectivity problems after setup, see the [MCP Troubleshooting Guide](./mcp_troubleshoot.md).
<Tabs>
<TabItem value="ui" label="LiteLLM UI">
@@ -326,6 +328,7 @@ litellm_settings:
</TabItem>
</Tabs>
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
@@ -502,7 +505,7 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
This configuration is currently available on the config.yaml, with UI support coming soon.
You can configure this either in `config.yaml` or directly from the LiteLLM UI (MCP Servers → Authentication → OAuth).
```yaml
mcp_servers:
@@ -1473,3 +1476,17 @@ async with stdio_client(server_params) as (read, write):
</TabItem>
</Tabs>
## FAQ
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**
The UI keeps only transient state in `sessionStorage` so the OAuth redirect flow can finish; the token is not persisted in the server or database.
**Q: I'm seeing MCP connection errors—what should I check?**
Walk through the [MCP Troubleshooting Guide](./mcp_troubleshoot.md) for step-by-step isolation (Client → LiteLLM vs. LiteLLM → MCP), log examples, and verification methods like MCP Inspector and `curl`.
docs/my-website/docs/mcp_troubleshoot.md
+99
View File
@@ -0,0 +1,99 @@
import Image from '@theme/IdealImage';
# MCP Troubleshooting Guide
When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Proxy → MCP Server`, while OAuth-enabled setups add an authorization server for metadata discovery.
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
## Locate the Error Source
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
<Image
img={require('../img/mcp_tool_testing_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
**Actions**
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
### Client Traffic Issues (Client → LiteLLM)
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
#### MCP Protocol Sessions
Clients such as IDEs or agent runtimes speak the MCP protocol directly with LiteLLM.
**Actions**
- Inspect LiteLLM access logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to verify the client request reached the proxy and which MCP server it targeted.
- Review LiteLLM error logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) for TLS, authentication, or routing errors that block the request before the MCP call starts.
- Use the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to confirm the MCP server is reachable outside of the failing client.
#### Responses/Completions with Embedded MCP Calls
During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls mid-request. An error could occur before the MCP call begins or after the MCP responds.
**Actions**
- Check LiteLLM request logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to see whether an MCP attempt was recorded; if not, the problem lies in `Client → LiteLLM`.
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
<Image
img={require('../img/mcp_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
### OAuth Metadata Discovery
LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://modelcontextprotocol.info/specification/draft/basic/authorization/#23-server-metadata-discovery)). When OAuth is enabled, confirm the authorization server exposes the metadata URL and that LiteLLM can fetch it.
**Actions**
- Use `curl <metadata_url>` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
## Verify Connectivity
Run lightweight validations before impacting production traffic.
### MCP Inspector
Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Client → MCP` communications in one place; it makes isolating the failing hop straightforward.
1. Execute `npx @modelcontextprotocol/inspector` on your workstation.
2. Configure and connect:
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
- **Custom Headers:** e.g., `Authorization: Bearer <LiteLLM API Key>`.
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
### `curl` Smoke Test
`curl` is ideal on servers where installing the Inspector is impractical. It replicates the MCP tool call LiteLLM would make—swap in the domain of the system under test (LiteLLM or the MCP server).
```bash
curl -X POST https://your-target-domain.example.com/mcp \
-H "Content-Type: application/json" \
-H "Accept: application/json, text/event-stream" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
Add `-H "Authorization: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
## Review Logs
Well-scoped logs make it clear whether LiteLLM reached the MCP server and what happened next.
### Access Log Example (successful MCP call)
```text
INFO: 127.0.0.1:57230 - "POST /everything/mcp HTTP/1.1" 200 OK
```
### Error Log Example (failed MCP call)
```text
07:22:00 - LiteLLM:ERROR: client.py:224 - MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception), Server: http://localhost:3001/mcp, Transport: MCPTransport.http
httpcore.ConnectError: All connection attempts failed
ERROR:LiteLLM:MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception)...
httpx.ConnectError: All connection attempts failed
```
docs/my-website/docs/observability/logfire_integration.md
+4
View File
@@ -40,6 +40,10 @@ import os
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_TOKEN"] = ""
# Optionally customize the base url
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_BASE_URL"] = ""
# LLM API Keys
os.environ['OPENAI_API_KEY']=""
docs/my-website/docs/providers/azure_ai/azure_model_router.md
+232
View File
@@ -0,0 +1,232 @@
# Azure Model Router
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
## LiteLLM Python SDK
### Basic Usage
```python
import litellm
import os
response = litellm.completion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
)
print(response)
```
### Streaming with Usage Tracking
```python
import litellm
import os
response = await litellm.acompletion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "hi"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
stream=True,
stream_options={"include_usage": True},
)
async for chunk in response:
print(chunk)
```
## LiteLLM Proxy (AI Gateway)
### config.yaml
```yaml
model_list:
- model_name: azure-model-router
litellm_params:
model: azure_ai/azure-model-router
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
```
### Start Proxy
```bash
litellm --config config.yaml
```
### Test Request
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "azure-model-router",
"messages": [{"role": "user", "content": "Hello!"}]
}'
```
## Add Azure Model Router via LiteLLM UI
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
### Select Provider
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
#### Navigate to Models Page
![Navigate to Models](./img/azure_model_router_01.jpeg)
#### Click Provider Dropdown
![Click Provider](./img/azure_model_router_02.jpeg)
#### Choose Azure AI Foundry
![Select Azure AI Foundry](./img/azure_model_router_03.jpeg)
### Configure Model Name
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
#### Click Model Name Field
![Click Model Field](./img/azure_model_router_04.jpeg)
#### Select Custom Model Name
![Select Custom Model](./img/azure_model_router_05.jpeg)
#### Enter LiteLLM Model Name
![LiteLLM Model Name](./img/azure_model_router_06.jpeg)
#### Click Custom Model Name Field
![Enter Custom Name Field](./img/azure_model_router_07.jpeg)
#### Type Model Prefix
Type `azure_ai/` as the prefix.
![Type azure_ai prefix](./img/azure_model_router_08.jpeg)
#### Copy Model Name from Azure Portal
Switch to Azure AI Foundry and copy your model router deployment name.
![Azure Portal Model Name](./img/azure_model_router_09.jpeg)
![Copy Model Name](./img/azure_model_router_10.jpeg)
#### Paste Model Name
Paste to get `azure_ai/azure-model-router`.
![Paste Model Name](./img/azure_model_router_11.jpeg)
### Configure API Base and Key
Copy the endpoint URL and API key from Azure portal.
#### Copy API Base URL from Azure
![Copy API Base](./img/azure_model_router_12.jpeg)
#### Enter API Base in LiteLLM
![Click API Base Field](./img/azure_model_router_13.jpeg)
![Paste API Base](./img/azure_model_router_14.jpeg)
#### Copy API Key from Azure
![Copy API Key](./img/azure_model_router_15.jpeg)
#### Enter API Key in LiteLLM
![Enter API Key](./img/azure_model_router_16.jpeg)
### Test and Add Model
Verify your configuration works and save the model.
#### Test Connection
![Test Connection](./img/azure_model_router_17.jpeg)
#### Close Test Dialog
![Close Dialog](./img/azure_model_router_18.jpeg)
#### Add Model
![Add Model](./img/azure_model_router_19.jpeg)
### Verify in Playground
Test your model and verify cost tracking is working.
#### Open Playground
![Go to Playground](./img/azure_model_router_20.jpeg)
#### Select Model
![Select Model](./img/azure_model_router_21.jpeg)
#### Send Test Message
![Send Message](./img/azure_model_router_22.jpeg)
#### View Logs
![View Logs](./img/azure_model_router_23.jpeg)
#### Verify Cost Tracking
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
![Verify Cost](./img/azure_model_router_24.jpeg)
## Cost Tracking
LiteLLM automatically handles cost tracking for Azure Model Router by:
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### Example Response with Cost
```python
import litellm
response = litellm.completion(
model="azure_ai/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
# The response will show the actual model used
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
# Get cost
from litellm import completion_cost
cost = completion_cost(completion_response=response)
print(f"Cost: ${cost}")
```
docs/my-website/docs/providers/azure_ai/img/azure_model_router_01.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 254 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_02.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 240 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_03.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 239 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_04.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 247 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_05.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 272 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_06.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 307 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_07.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 248 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_08.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 315 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_09.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 492 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_10.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 562 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_11.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 265 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_12.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 471 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_13.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 225 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_14.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 219 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_15.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 485 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_16.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 227 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_17.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 236 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_18.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 214 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_19.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 232 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_20.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 256 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_21.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 250 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_22.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 206 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_23.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 311 KiB

docs/my-website/docs/providers/azure_ai/img/azure_model_router_24.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 419 KiB

docs/my-website/docs/providers/chatgpt.md
+84
View File
@@ -0,0 +1,84 @@
# ChatGPT Subscription
Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow authentication.
| Property | Details |
|-------|-------|
| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
| Provider Route on LiteLLM | `chatgpt/` |
| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
| API Reference | https://chatgpt.com |
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
Notes:
- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
- `/v1/chat/completions` honors `stream`. When `stream` is false (default), LiteLLM aggregates the Responses stream into a single JSON response.
## Authentication
ChatGPT subscription access uses an OAuth device code flow:
1. LiteLLM prints a device code and verification URL
2. Open the URL, sign in, and enter the code
3. Tokens are stored locally for reuse
## Usage - LiteLLM Python SDK
### Responses (recommended for Codex models)
```python showLineNumbers title="ChatGPT Responses"
import litellm
response = litellm.responses(
model="chatgpt/gpt-5.2-codex",
input="Write a Python hello world"
)
print(response)
```
### Chat Completions (bridged to Responses)
```python showLineNumbers title="ChatGPT Chat Completions"
import litellm
response = litellm.completion(
model="chatgpt/gpt-5.2",
messages=[{"role": "user", "content": "Write a Python hello world"}]
)
print(response)
```
## Usage - LiteLLM Proxy
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: chatgpt/gpt-5.2
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.2-codex
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2-codex
```
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
```
## Configuration
### Environment Variables
- `CHATGPT_TOKEN_DIR`: Custom token storage directory
- `CHATGPT_AUTH_FILE`: Auth file name (default: `auth.json`)
- `CHATGPT_API_BASE`: Override API base (default: `https://chatgpt.com/backend-api/codex`)
- `OPENAI_CHATGPT_API_BASE`: Alias for `CHATGPT_API_BASE`
- `CHATGPT_ORIGINATOR`: Override the `originator` header value
- `CHATGPT_USER_AGENT`: Override the `User-Agent` header value
- `CHATGPT_USER_AGENT_SUFFIX`: Optional suffix appended to the `User-Agent` header
docs/my-website/docs/providers/openrouter.md
+117
View File
@@ -93,3 +93,120 @@ response = embedding(
)
print(response)
```
## Image Generation
OpenRouter supports image generation through select models like Google Gemini image generation models. LiteLLM transforms standard image generation requests to OpenRouter's chat completion format.
### Supported Parameters
- `size`: Maps to OpenRouter's `aspect_ratio` format
- `1024x1024` → `1:1` (square)
- `1536x1024` → `3:2` (landscape)
- `1024x1536` → `2:3` (portrait)
- `1792x1024` → `16:9` (wide landscape)
- `1024x1792` → `9:16` (tall portrait)
- `quality`: Maps to OpenRouter's `image_size` format (Gemini models)
- `low` or `standard` → `1K`
- `medium` → `2K`
- `high` or `hd` → `4K`
- `n`: Number of images to generate
### Usage
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Basic image generation
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
)
print(response)
```
### Advanced Usage with Parameters
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Generate high-quality landscape image
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A serene mountain landscape with a lake",
size="1536x1024", # Landscape format
quality="high", # High quality (4K)
)
# Access the generated image
image_data = response.data[0]
if image_data.b64_json:
# Base64 encoded image
print(f"Generated base64 image: {image_data.b64_json[:50]}...")
elif image_data.url:
# Image URL
print(f"Generated image URL: {image_data.url}")
```
### Using OpenRouter-Specific Parameters
You can also pass OpenRouter-specific parameters directly using `image_config`:
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A futuristic cityscape at night",
image_config={
"aspect_ratio": "16:9", # OpenRouter native format
"image_size": "4K" # OpenRouter native format
}
)
print(response)
```
### Response Format
The response follows the standard LiteLLM ImageResponse format:
```python
{
"created": 1703658209,
"data": [{
"b64_json": "iVBORw0KGgoAAAANSUhEUgAA...", # Base64 encoded image
"url": None,
"revised_prompt": None
}],
"usage": {
"input_tokens": 10,
"output_tokens": 1290,
"total_tokens": 1300
}
}
```
### Cost Tracking
OpenRouter provides cost information in the response, which LiteLLM automatically tracks:
```python
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A cute baby sea otter",
)
# Cost is available in the response metadata
print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
```
docs/my-website/docs/providers/sap.md
+466 -69
View File
@@ -12,100 +12,340 @@ LiteLLM supports SAP Generative AI Hub's Orchestration Service.
| Supported Endpoints | `/chat/completions`, `/embeddings` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
## Prerequisites
Before you begin, ensure you have:
1. **SAP BTP Account** with access to SAP AI Core
2. **AI Core Service Instance** provisioned in your subaccount
3. **Service Key** created for your AI Core instance (this contains your credentials)
4. **Resource Group** with deployed AI models (check with your SAP administrator)
:::tip Where to Find Your Credentials
Your credentials come from the **Service Key** you create in SAP BTP Cockpit:
1. Navigate to your **Subaccount** → **Instances and Subscriptions**
2. Find your **AI Core** instance and click on it
3. Go to **Service Keys** and create one (or use existing)
4. The JSON contains all values needed below
The service key JSON looks like this:
```json
{
"clientid": "sb-abc123...",
"clientsecret": "xyz789...",
"url": "https://myinstance.authentication.eu10.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.prod.eu-central-1.aws.ml.hana.ondemand.com"
}
}
```
:::info Resource Group
The resource group is typically configured separately in your AI Core deployment, not in the service key itself. You can set it via the `AICORE_RESOURCE_GROUP` environment variable (defaults to "default").
:::
## Quick Start
### Step 1: Install LiteLLM
```bash
pip install litellm
```
### Step 2: Set Your Credentials
Choose **one** of these authentication methods:
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
The simplest approach - paste your entire service key as a single environment variable. The service key must be wrapped in a `credentials` object:
```bash
export AICORE_SERVICE_KEY='{
"credentials": {
"clientid": "your-client-id",
"clientsecret": "your-client-secret",
"url": "https://<your-instance>.authentication.sap.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.<your-region>.aws.ml.hana.ondemand.com"
}
}
}'
export AICORE_RESOURCE_GROUP="default"
```
</TabItem>
<TabItem value="individual" label="Individual Variables">
Alternatively, instead of using the service key above, you could set each credential separately:
```bash
export AICORE_AUTH_URL="https://<your-instance>.authentication.sap.hana.ondemand.com/oauth/token"
export AICORE_CLIENT_ID="your-client-id"
export AICORE_CLIENT_SECRET="your-client-secret"
export AICORE_RESOURCE_GROUP="default"
export AICORE_BASE_URL="https://api.ai.<your-region>.aws.ml.hana.ondemand.com/v2"
```
</TabItem>
</Tabs>
### Step 3: Make Your First Request
```python title="test_sap.py"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello from LiteLLM!"}]
)
print(response.choices[0].message.content)
```
Run it:
```bash
python test_sap.py
```
**Expected output:**
```text
Hello! How can I assist you today?
```
### Step 4: Verify Your Setup (Optional)
Test that everything is working with this diagnostic script:
```python title="verify_sap_setup.py"
import os
import litellm
# Enable debug logging to see what's happening
import os
os.environ["LITELLM_LOG"] = "DEBUG"
# Either use AICORE_SERVICE_KEY (contains all credentials including resourcegroup)
# OR use individual variables (all required together)
individual_vars = ["AICORE_AUTH_URL", "AICORE_CLIENT_ID", "AICORE_CLIENT_SECRET", "AICORE_BASE_URL", "AICORE_RESOURCE_GROUP"]
print("=== SAP Gen AI Hub Setup Verification ===\n")
# Check for service key method
if os.environ.get("AICORE_SERVICE_KEY"):
print("✓ Using AICORE_SERVICE_KEY authentication (includes resource group)")
else:
# Check individual variables
missing = [v for v in individual_vars if not os.environ.get(v)]
if missing:
print(f"✗ Missing environment variables: {missing}")
else:
print("✓ Using individual variable authentication")
print(f"✓ Resource group: {os.environ.get('AICORE_RESOURCE_GROUP')}")
# Test API connection
print("\n=== Testing API Connection ===\n")
try:
response = litellm.completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Say 'Connection successful!' and nothing else."}],
max_tokens=20
)
print(f"✓ API Response: {response.choices[0].message.content}")
print("\n🎉 Setup complete! You're ready to use SAP Gen AI Hub with LiteLLM.")
except Exception as e:
print(f"✗ API Error: {e}")
print("\nTroubleshooting tips:")
print(" 1. Verify your service key credentials are correct")
print(" 2. Check that 'gpt-4o' is deployed in your resource group")
print(" 3. Ensure your SAP AI Core instance is running")
```
Run the verification:
```bash
python verify_sap_setup.py
```
**Expected output on success:**
```text
=== SAP Gen AI Hub Setup Verification ===
✓ Using AICORE_SERVICE_KEY authentication
✓ Resource group: default
=== Testing API Connection ===
✓ API Response: Connection successful!
🎉 Setup complete! You're ready to use SAP Gen AI Hub with LiteLLM.
```
## Authentication
SAP Generative AI Hub uses service key authentication. You can provide credentials via:
SAP Generative AI Hub uses OAuth2 service keys for authentication. See [Quick Start](#quick-start) for setup instructions.
1. **Environment variable** - Set `AICORE_SERVICE_KEY` with your service key JSON
2. **Direct parameter** - Pass `api_key` with the service key JSON string
### Environment Variables Reference
```python showLineNumbers title="Environment Variable"
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
| Variable | Required | Description |
|----------|----------|-------------|
| `AICORE_SERVICE_KEY` | Yes* | Complete service key JSON (recommended method) |
| `AICORE_RESOURCE_GROUP` | Yes | Your AI Core resource group name |
| `AICORE_AUTH_URL` | Yes* | OAuth token URL (alternative to service key) |
| `AICORE_CLIENT_ID` | Yes* | OAuth client ID (alternative to service key) |
| `AICORE_CLIENT_SECRET` | Yes* | OAuth client secret (alternative to service key) |
| `AICORE_BASE_URL` | Yes* | AI Core API base URL (alternative to service key) |
*Choose either `AICORE_SERVICE_KEY` OR the individual variables (`AICORE_AUTH_URL`, `AICORE_CLIENT_ID`, `AICORE_CLIENT_SECRET`, `AICORE_BASE_URL`).
## Model Naming Conventions
Understanding model naming is crucial for using SAP Gen AI Hub correctly. The naming pattern differs depending on whether you're using the SDK directly or through the proxy.
### Direct SDK Usage
When calling LiteLLM's SDK directly, you **must** include the `sap/` prefix in the model name:
```python
# Correct - includes sap/ prefix
model="sap/gpt-4o"
model="sap/anthropic--claude-4.5-sonnet"
model="sap/gemini-2.5-pro"
# Incorrect - missing prefix
model="gpt-4o" # ❌ Won't work
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
## Usage - LiteLLM Python SDK
```python showLineNumbers title="SAP Chat Completion"
from litellm import completion
import os
### Proxy Usage
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
When using the LiteLLM Proxy, you use the **friendly `model_name`** defined in your configuration. The proxy automatically handles the `sap/` prefix routing.
response = completion(
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello from LiteLLM"}]
```yaml
# In config.yaml, define the mapping
model_list:
- model_name: gpt-4o # ← Use this name in client requests
litellm_params:
model: sap/gpt-4o # ← Proxy handles the sap/ prefix
```
```python
# Client request - no sap/ prefix needed
client.chat.completions.create(
model="gpt-4o", # ✓ Correct for proxy usage
messages=[...]
)
print(response)
```
```python showLineNumbers title="SAP Chat Completion - Streaming"
### Anthropic Models Special Syntax
Anthropic models use a double-dash (`--`) prefix convention:
| Provider | Model Example | LiteLLM Format |
|----------|---------------|----------------|
| OpenAI | GPT-4o | `sap/gpt-4o` |
| Anthropic | Claude 4.5 Sonnet | `sap/anthropic--claude-4.5-sonnet` |
| Google | Gemini 2.5 Pro | `sap/gemini-2.5-pro` |
| Mistral | Mistral Large | `sap/mistral-large` |
### Quick Reference Table
| Usage Type | Model Format | Example |
|------------|--------------|---------|
| Direct SDK | `sap/<model-name>` | `sap/gpt-4o` |
| Direct SDK (Anthropic) | `sap/anthropic--<model>` | `sap/anthropic--claude-4.5-sonnet` |
| Proxy Client | `<friendly-name>` | `gpt-4o` or `claude-sonnet` |
## Using the Python SDK
The LiteLLM Python SDK automatically detects your authentication method. Simply set your environment variables and make requests.
```python showLineNumbers title="Basic Completion"
from litellm import completion
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
# Assumes AICORE_AUTH_URL, AICORE_CLIENT_ID, etc. are set
response = completion(
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
stream=True
model="sap/anthropic--claude-4.5-sonnet",
messages=[{"role": "user", "content": "Explain quantum computing"}]
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
print(response.choices[0].message.content)
```
```python showLineNumbers title="SAP Embedding"
from litellm import embedding
import os
Both authentication methods (individual variables or service key JSON) work automatically - no code changes required.
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
## Using the Proxy Server
result = embedding(
model="sap/text-embedding-3-small",
input="Answer to the ultimate question of life, the universe, and everything is 42")
print(result.data[0])
```
The LiteLLM Proxy provides a unified OpenAI-compatible API for your SAP models.
## Usage - LiteLLM Proxy
### Configuration
Add to your LiteLLM Proxy config:
Create a `config.yaml` file in your project directory with your model mappings and credentials:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: "sap/*"
# OpenAI models
- model_name: gpt-5
litellm_params:
model: "sap/*"
model: sap/gpt-5
general_settings:
master_key: your-proxy-api-key
# Anthropic models (note the double-dash)
- model_name: claude-sonnet
litellm_params:
model: sap/anthropic--claude-4.5-sonnet
- model_name: claude-opus
litellm_params:
model: sap/anthropic--claude-4.5-opus
# Embeddings
- model_name: text-embedding-3-small
litellm_params:
model: sap/text-embedding-3-small
litellm_settings:
drop_params: true
set_verbose: false
request_timeout: 600
num_retries: 2
forward_client_headers_to_llm_api: ["anthropic-version"]
general_settings:
master_key: "sk-1234" # Enter here your desired master key starting with 'sk-'.
# UI Admin is not required but helpful including the management of keys for your team(s). If you are using a database, these parameters are required:
database_url: "Enter you database URL."
UI_USERNAME: "Your desired UI admin account name"
UI_PASSWORD: "Your desired and strong pwd"
# Authentication
environment_variables:
AICORE_SERVICE_KEY: '{"clientid": "...", "clientsecret": "...", ...}'
AICORE_SERVICE_KEY: '{"credentials": {"clientid": "...", "clientsecret": "...", "url": "...", "serviceurls": {"AI_API_URL": "..."}}}'
AICORE_RESOURCE_GROUP: "default"
```
Start the proxy:
### Starting the Proxy
```bash showLineNumbers title="Start Proxy"
litellm --config config.yaml
```
The proxy will start on `http://localhost:4000` by default.
### Making Requests
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Test Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "sap/gpt-4",
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
@@ -118,11 +358,11 @@ from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-api-key"
api_key="sk-1234"
)
response = client.chat.completions.create(
model="sap/gpt-4",
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
print(response.choices[0].message.content)
@@ -134,12 +374,14 @@ print(response.choices[0].message.content)
```python showLineNumbers title="LiteLLM SDK"
import os
import litellm
os.environ["LITELLM_PROXY_API_KEY"] = "your-proxy-api-key"
litellm.use_litellm_proxy = True # it is important to set this parameter
os.environ["LITELLM_PROXY_API_KEY"] = "sk-1234"
litellm.use_litellm_proxy = True
response = litellm.completion(
model="sap/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="http://your-proxy-api-base"
model="claude-sonnet",
messages=[{"content": "Hello, how are you?", "role": "user"}],
api_base="http://localhost:4000"
)
print(response)
@@ -148,15 +390,170 @@ print(response)
</TabItem>
</Tabs>
## Supported Parameters
## Features
| Parameter | Description |
|-----------|-------------|
| `temperature` | Controls randomness |
| `max_tokens` | Maximum tokens in response |
| `top_p` | Nucleus sampling |
| `tools` | Function calling tools |
| `tool_choice` | Tool selection behavior |
| `response_format` | Output format (json_object, json_schema) |
| `stream` | Enable streaming |
### Streaming Responses
Stream responses in real-time for better user experience:
```python showLineNumbers title="Streaming Chat Completion"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Count from 1 to 10"}],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
```
### Structured Output
#### JSON Schema (Recommended)
Use JSON Schema for structured output with strict validation:
```python showLineNumbers title="JSON Schema Response"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{
"role": "user",
"content": "Generate info about Tokyo"
}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "city_info",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"population": {"type": "number"},
"country": {"type": "string"}
},
"required": ["name", "population", "country"],
"additionalProperties": False
},
"strict": True
}
}
)
print(response.choices[0].message.content)
# Output: {"name":"Tokyo","population":37000000,"country":"Japan"}
```
#### JSON Object Format
For flexible JSON output without schema validation:
```python showLineNumbers title="JSON Object Response"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{
"role": "user",
"content": "Generate a person object in JSON format with name and age"
}],
response_format={"type": "json_object"}
)
print(response.choices[0].message.content)
```
:::note SAP Platform Requirement
When using `json_object` type, SAP's orchestration service requires the word "json" to appear in your prompt. This ensures explicit intent for JSON formatting. For schema-validated output without this requirement, use `json_schema` instead (recommended).
:::
### Multi-turn Conversations
Maintain conversation context across multiple turns:
```python showLineNumbers title="Multi-turn Conversation"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[
{"role": "user", "content": "My name is Alice"},
{"role": "assistant", "content": "Hello Alice! Nice to meet you."},
{"role": "user", "content": "What is my name?"}
]
)
print(response.choices[0].message.content)
# Output: Your name is Alice.
```
### Embeddings
Generate vector embeddings for semantic search and retrieval:
```python showLineNumbers title="Create Embeddings"
from litellm import embedding
response = embedding(
model="sap/text-embedding-3-small",
input=["Hello world", "Machine learning is fascinating"]
)
print(response.data[0]["embedding"]) # Vector representation
```
## Reference
### Supported Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `model` | string | Model identifier (with `sap/` prefix for SDK) |
| `messages` | array | Conversation messages |
| `temperature` | float | Controls randomness (0-2) |
| `max_tokens` | integer | Maximum tokens in response |
| `top_p` | float | Nucleus sampling threshold |
| `stream` | boolean | Enable streaming responses |
| `response_format` | object | Output format (`json_object`, `json_schema`) |
| `tools` | array | Function calling tool definitions |
| `tool_choice` | string/object | Tool selection behavior |
### Supported Models
For the complete and up-to-date list of available models provided by SAP Gen AI Hub, please refer to the [SAP AI Core Generative AI Hub documentation](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/models-and-scenarios-in-generative-ai-hub).
:::info Model Availability
Model availability varies by SAP deployment region and your subscription. Contact your SAP administrator to confirm which models are available in your environment.
:::
### Troubleshooting
**Authentication Errors**
If you receive authentication errors:
1. Verify all required environment variables are set correctly
2. Check that your service key hasn't expired
3. Confirm your resource group has access to the desired models
4. Ensure the `AICORE_AUTH_URL` and `AICORE_BASE_URL` match your SAP region
**Model Not Found**
If a model returns "not found":
1. Verify the model is available in your SAP deployment
2. Check you're using the correct model name format (`sap/` prefix for SDK)
3. Confirm your resource group has access to that specific model
4. For Anthropic models, ensure you're using the `anthropic--` double-dash prefix
**Rate Limiting**
SAP Gen AI Hub enforces rate limits based on your subscription. If you hit limits:
1. Implement exponential backoff retry logic
2. Consider using the proxy's built-in rate limiting features
3. Contact your SAP administrator to review quota allocations
docs/my-website/docs/providers/stability.md
-1
View File
@@ -416,7 +416,6 @@ response = image_edit(
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add flowers in the masked area",
size="1024x1024",
)
print(response)
```
docs/my-website/docs/proxy/caching.md
+15
View File
@@ -282,6 +282,10 @@ Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable cac
```
**Additional kwargs**
:::info
Use `REDIS_*` environment variables to configure all Redis client library parameters. This is the suggested mechanism for toggling Redis settings as it automatically maps environment variables to Redis client kwargs.
:::
You can pass in any additional redis.Redis arg, by storing the variable + value in your os
environment, like this:
@@ -289,6 +293,17 @@ environment, like this:
REDIS_<redis-kwarg-name> = ""
```
For example:
```shell
REDIS_SSL = "True"
REDIS_SSL_CERT_REQS = "None"
REDIS_CONNECTION_POOL_KWARGS = '{"max_connections": 20}'
```
:::warning
**Note**: For non-string Redis parameters (like integers, booleans, or complex objects), avoid using `REDIS_*` environment variables as they may fail during Redis client initialization. Instead, use `cache_kwargs` in your router configuration for such parameters.
:::
[**See how it's read from the environment**](https://github.com/BerriAI/litellm/blob/4d7ff1b33b9991dcf38d821266290631d9bcd2dd/litellm/_redis.py#L40)
#### Step 3: Run proxy with config
docs/my-website/docs/proxy/config_settings.md
+3 -1
View File
@@ -339,7 +339,7 @@ router_settings:
| stream_timeout | Optional[float] | The default timeout for a streaming request. If not set, the 'timeout' value is used. |
| debug_level | Literal["DEBUG", "INFO"] | The debug level for the logging library in the router. Defaults to "INFO". |
| client_ttl | int | Time-to-live for cached clients in seconds. Defaults to 3600. |
| cache_kwargs | dict | Additional keyword arguments for the cache initialization. |
| cache_kwargs | dict | Additional keyword arguments for the cache initialization. Use this for non-string Redis parameters that may fail when set via `REDIS_*` environment variables. |
| routing_strategy_args | dict | Additional keyword arguments for the routing strategy - e.g. lowest latency routing default ttl |
| model_group_alias | dict | Model group alias mapping. E.g. `{"claude-3-haiku": "claude-3-haiku-20240229"}` |
| num_retries | int | Number of retries for a request. Defaults to 3. |
@@ -744,6 +744,7 @@ router_settings:
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
| LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
| LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
@@ -754,6 +755,7 @@ router_settings:
| LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS | Cooldown time in seconds before allowing another aggressive clear operation when the queue is full. Default is 0.5
| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_IMAGE_URL_DOWNLOAD_SIZE_MB | Maximum size in MB for downloading images from URLs. Prevents memory issues from downloading very large images. Images exceeding this limit will be rejected before download. Set to 0 to completely disable image URL handling (all image_url requests will be blocked). Default is 50MB (matching [OpenAI's limit](https://platform.openai.com/docs/guides/images-vision?api-mode=chat#image-input-requirements))
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100
| MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768
docs/my-website/docs/proxy/custom_pricing.md
-46
View File
@@ -9,7 +9,6 @@ LiteLLM provides flexible cost tracking and pricing customization for all LLM pr
- **Custom Pricing** - Override default model costs or set pricing for custom models
- **Cost Per Token** - Track costs based on input/output tokens (most common)
- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker)
- **Zero-Cost Models** - Bypass budget checks for free/on-premises models by setting costs to 0
- **[Provider Discounts](./provider_discounts.md)** - Apply percentage-based discounts to specific providers
- **[Provider Margins](./provider_margins.md)** - Add fees/margins to LLM costs for internal billing
- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments
@@ -107,51 +106,6 @@ There are other keys you can use to specify costs for different scenarios and mo
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
## Zero-Cost Models (Bypass Budget Checks)
**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.
**Solution** ✅: Set both `input_cost_per_token` and `output_cost_per_token` to `0` (explicitly) to bypass all budget checks for that model.
:::info
When a model is configured with zero cost, LiteLLM will automatically skip ALL budget checks (user, team, team member, end-user, organization, and global proxy budget) for requests to that model.
**Important**: Both costs must be **explicitly set to 0**. If costs are `null` or undefined, the model will be treated as having cost and budget checks will apply.
:::
### Configuration Example
```yaml
model_list:
# On-premises model - free to use
- model_name: on-prem-llama
litellm_params:
model: ollama/llama3
api_base: http://localhost:11434
model_info:
input_cost_per_token: 0 # 👈 Explicitly set to 0
output_cost_per_token: 0 # 👈 Explicitly set to 0
# Paid cloud model - budget checks apply
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
# No model_info - uses default pricing from cost map
```
### Behavior
With the above configuration:
- **User over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **Team over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **End-user over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
This ensures your free/on-premises models remain accessible regardless of budget constraints, while paid models are still properly governed.
## Set 'base_model' for Cost Tracking (e.g. Azure deployments)
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking
docs/my-website/docs/proxy/customer_usage.md
+51 -6
View File
@@ -22,19 +22,22 @@ Customer Usage enables you to track spend and usage for individual customers (en
## How to Track Spend
Track customer spend by including a `user` field in your API requests. The customer ID will be automatically tracked and associated with all spend from that request.
Track customer spend by including a `user` field in your API requests or by passing a customer ID header. The customer ID will be automatically tracked and associated with all spend from that request.
### Example using cURL
<Tabs>
<TabItem value="body" label="Request Body" default>
### Using Request Body
Make a `/chat/completions` call with the `user` field containing your customer ID:
```bash showLineNumbers title="Track spend with customer ID"
```bash showLineNumbers title="Track spend with customer ID in body"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gpt-3.5-turbo",
"user": "customer-123", # 👈 CUSTOMER ID
"user": "customer-123",
"messages": [
{
"role": "user",
@@ -44,7 +47,49 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
}'
```
The customer ID (`customer-123`) will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented.
</TabItem>
<TabItem value="header" label="Request Header">
### Using Request Headers
You can also pass the customer ID via HTTP headers. This is useful for tools that support custom headers but don't allow modifying the request body (like Claude Code with `ANTHROPIC_CUSTOM_HEADERS`).
LiteLLM automatically recognizes these standard headers (no configuration required):
- `x-litellm-customer-id`
- `x-litellm-end-user-id`
```bash showLineNumbers title="Track spend with customer ID in header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'x-litellm-customer-id: customer-123' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
#### Using with Claude Code
Claude Code supports custom headers via the `ANTHROPIC_CUSTOM_HEADERS` environment variable. Set it to pass your customer ID:
```bash title="Configure Claude Code with customer tracking"
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/v1/messages"
export ANTHROPIC_API_KEY="sk-1234"
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-customer-id: my-customer-id"
```
Now all requests from Claude Code will automatically track spend under `my-customer-id`.
</TabItem>
</Tabs>
The customer ID will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented.
### Example using OpenWebUI
docs/my-website/docs/proxy/deleted_keys_teams.md
+106
View File
@@ -0,0 +1,106 @@
import Image from '@theme/IdealImage';
# Deleted Keys & Teams Audit Logs
<Image img={require('../../img/ui_deleted_keys_table.png')} />
View deleted API keys and teams along with their spend and budget information at the time of deletion for auditing and compliance purposes.
## Overview
The Deleted Keys & Teams feature provides a comprehensive audit trail for deleted entities in your LiteLLM proxy. This feature was implemented to easily allow audits of which key or team was deleted along with the spend/budget at the time of deletion.
When a key or team is deleted, LiteLLM automatically captures:
- **Deletion timestamp** - When the entity was deleted
- **Deleted by** - Who performed the deletion action
- **Spend at deletion** - The total spend accumulated at the time of deletion
- **Original budget** - The budget that was set for the entity before deletion
- **Entity details** - Key or team identification information
This information is preserved even after deletion, allowing you to maintain accurate financial records and audit trails for compliance purposes.
## Viewing Deleted Keys
### Step 1: Navigate to API Keys Page
Navigate to the API Keys page in the LiteLLM UI:
```
http://localhost:4000/ui/?login=success&page=api-keys
```
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/73b97ba9-0ab5-4140-aee2-05fa90463461/ascreenshot_5e6d9f05d452405c83d7a368349d087d_text_export.jpeg)
### Step 2: Access Logs Section
Click on the "Logs" menu item in the navigation.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/73b97ba9-0ab5-4140-aee2-05fa90463461/ascreenshot_8ebab354b1e542e59e1082e519927edd_text_export.jpeg)
### Step 3: View Deleted Keys
Click on "Deleted Keys" to view the table of all deleted API keys.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/00668558-9326-4a6f-8e87-159d54b17a72/ascreenshot_d0e50e49e9aa43d4a22ada6f12a78b12_text_export.jpeg)
### Step 4: Review Deletion Information
The Deleted Keys table includes comprehensive information about each deleted key:
- **When** the key was deleted (timestamp)
- **Who** deleted the key (user/admin information)
- **Key identification** details
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/8538f7c4-634e-44c8-8d7d-fafbd6da0b02/ascreenshot_6b73f9c6a52d4e40a2368ef441cf6c8f_text_export.jpeg)
### Step 5: View Financial Information
The table also displays financial information captured at the time of deletion:
- **Spend at deletion** - Total spend accumulated when the key was deleted
- **Original budget** - The budget limit that was set for the key
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/f8b03850-b17c-490c-a507-c3b0b6c050ab/ascreenshot_070b139f111844bba38fbed8835b097b_text_export.jpeg)
## Viewing Deleted Teams
### Step 1: Access Deleted Teams
From the Logs section, click on "Deleted Teams" to view all deleted teams.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/716ce26f-09af-4a6d-99c5-921d6b6a8555/ascreenshot_d36c16f1cf894340aa8bc20ada5922ac_text_export.jpeg)
### Step 2: Review Team Deletion Information
The Deleted Teams table provides detailed information about each deleted team:
- **When** the team was deleted (timestamp)
- **Who** deleted the team (user/admin information)
- **Team identification** details
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/0a3f2d3f-179a-4ad7-916e-b77a13dca01d/ascreenshot_ded5970762d54528ae656421148116c4_text_export.jpeg)
### Step 3: View Team Financial Information
Similar to deleted keys, the Deleted Teams table shows financial information:
- **Spend at deletion** - Total spend accumulated when the team was deleted
- **Original budget** - The budget limit that was set for the team
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/5b24871f-b57e-404d-8fbe-a4b27cb2a6a0/ascreenshot_3121fbafbd6b4abf90993ce6c03c608d_text_export.jpeg)
## Use Cases
This feature is particularly useful for:
- **Financial Auditing** - Track spend and budgets for deleted entities
- **Compliance** - Maintain records of who deleted what and when
- **Cost Analysis** - Understand spending patterns before deletion
- **Accountability** - Identify which admin or user performed deletions
- **Historical Records** - Preserve financial data even after entity deletion
## Related Features
- [Audit Logs](./multiple_admins.md) - View comprehensive audit logs for all entity changes
- [UI Logs](./ui_logs.md) - View request logs and spend tracking
docs/my-website/docs/proxy/fallback_management.md
+267
View File
@@ -0,0 +1,267 @@
# [New] Fallback Management Endpoints
Dedicated endpoints for managing model fallbacks separately from the general configuration.
## Overview
These endpoints allow you to configure, retrieve, and delete fallback models without modifying the entire proxy configuration. This provides a cleaner and safer way to manage fallbacks compared to using the `/config/update` endpoint.
## Prerequisites
- Database storage must be enabled: Set `STORE_MODEL_IN_DB=True` in your environment
- Models must exist in the router before configuring fallbacks
## Endpoints
### POST /fallback
Create or update fallbacks for a specific model.
**Request Body:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
**Parameters:**
- `model` (string, required): The primary model name to configure fallbacks for
- `fallback_models` (array of strings, required): List of fallback model names in priority order
- `fallback_type` (string, optional): Type of fallback. Options:
- `"general"` (default): Standard fallbacks for any error
- `"context_window"`: Fallbacks for context window exceeded errors
- `"content_policy"`: Fallbacks for content policy violations
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general",
"message": "Fallback configuration created successfully"
}
```
**Example using cURL:**
```bash
curl -X POST "http://localhost:4000/fallback" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}'
```
**Example using Python:**
```python
import requests
response = requests.post(
"http://localhost:4000/fallback",
headers={
"Authorization": "Bearer sk-1234",
"Content-Type": "application/json"
},
json={
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
)
print(response.json())
```
### GET /fallback/\{model\}
Get fallback configuration for a specific model.
**Parameters:**
- `model` (path parameter, required): The model name to get fallbacks for
- `fallback_type` (query parameter, optional): Type of fallback to retrieve (default: "general")
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
**Example using cURL:**
```bash
curl -X GET "http://localhost:4000/fallback/gpt-3.5-turbo?fallback_type=general" \
-H "Authorization: Bearer sk-1234"
```
**Example using Python:**
```python
import requests
response = requests.get(
"http://localhost:4000/fallback/gpt-3.5-turbo",
headers={"Authorization": "Bearer sk-1234"},
params={"fallback_type": "general"}
)
print(response.json())
```
### DELETE /fallback/\{model\}
Delete fallback configuration for a specific model.
**Parameters:**
- `model` (path parameter, required): The model name to delete fallbacks for
- `fallback_type` (query parameter, optional): Type of fallback to delete (default: "general")
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_type": "general",
"message": "Fallback configuration deleted successfully"
}
```
**Example using cURL:**
```bash
curl -X DELETE "http://localhost:4000/fallback/gpt-3.5-turbo?fallback_type=general" \
-H "Authorization: Bearer sk-1234"
```
**Example using Python:**
```python
import requests
response = requests.delete(
"http://localhost:4000/fallback/gpt-3.5-turbo",
headers={"Authorization": "Bearer sk-1234"},
params={"fallback_type": "general"}
)
print(response.json())
```
### Test fallback
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "ping"
}
],
"mock_testing_fallbacks": true
}
'
```
## Validation
The endpoints perform the following validations:
1. **Model Existence**: Verifies that the primary model exists in the router
2. **Fallback Model Existence**: Ensures all fallback models exist in the router
3. **No Self-Fallback**: Prevents a model from being its own fallback
4. **No Duplicates**: Ensures no duplicate models in the fallback list
5. **Database Enabled**: Requires `STORE_MODEL_IN_DB=True` to be set
## Error Responses
### 400 Bad Request
```json
{
"detail": {
"error": "Invalid fallback models: ['non-existent-model']",
"available_models": ["gpt-3.5-turbo", "gpt-4", "claude-3-haiku"]
}
}
```
### 404 Not Found
```json
{
"detail": {
"error": "Model 'gpt-3.5-turbo' not found in router",
"available_models": ["gpt-4", "claude-3-haiku"]
}
}
```
### 500 Internal Server Error
```json
{
"detail": {
"error": "Router not initialized"
}
}
```
## Fallback Types Explained
### General Fallbacks
Used for any type of error that occurs during model invocation. This is the most common type of fallback.
**Use Case:** When a model is unavailable, rate-limited, or returns an error.
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
### Context Window Fallbacks
Specifically triggered when a context window exceeded error occurs.
**Use Case:** When the input is too long for the primary model, fallback to a model with a larger context window.
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4-32k", "claude-3-opus"],
"fallback_type": "context_window"
}
```
### Content Policy Fallbacks
Specifically triggered when content policy violations occur.
**Use Case:** When the primary model rejects content due to safety filters, fallback to a model with different content policies.
```json
{
"model": "gpt-4",
"fallback_models": ["claude-3-haiku"],
"fallback_type": "content_policy"
}
```
## Benefits Over /config/update
1. **Safety**: Only modifies fallback configuration, won't accidentally change other settings
2. **Simplicity**: Focused API with clear validation messages
3. **Granularity**: Manage fallbacks per model and per type
4. **Validation**: Comprehensive checks ensure configuration is valid before applying
5. **Clarity**: Clear error messages with available models listed
## Notes
- Fallbacks are triggered after the configured number of retries fails
- Fallbacks are attempted in the order specified in `fallback_models`
- The maximum number of fallbacks attempted is controlled by the router's `max_fallbacks` setting
- Changes take effect immediately and are persisted to the database
docs/my-website/docs/proxy/guardrails/panw_prisma_airs.md
+28
View File
@@ -206,6 +206,7 @@ Expected successful response:
| `mode` | No | When to run the guardrail | `pre_call` |
| `fallback_on_error` | No | Action when PANW API is unavailable: `"block"` (fail-closed, default) or `"allow"` (fail-open). Config errors always block. | `block` |
| `timeout` | No | PANW API call timeout in seconds (1-60) | `10.0` |
| `violation_message_template` | No | Custom template for error message when request is blocked. Supports `{guardrail_name}`, `{category}`, `{action_type}`, `{default_message}` placeholders. | - |
### Regional Endpoints
@@ -449,6 +450,33 @@ LiteLLM does not alter or configure your PANW security profile. To change what c
The guardrail is **fail-closed** by default - if the PANW API is unavailable, requests are blocked to ensure no unscanned content reaches your LLM. This provides maximum security.
:::
### Custom Violation Messages
You can customize the error message returned to the user when a request is blocked by configuring the `violation_message_template` parameter. This is useful for providing user-friendly feedback instead of technical details.
```yaml
guardrails:
- guardrail_name: "panw-custom-message"
litellm_params:
guardrail: panw_prisma_airs
api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
# Simple message
violation_message_template: "Your request was blocked by our AI Security Policy."
- guardrail_name: "panw-detailed-message"
litellm_params:
guardrail: panw_prisma_airs
api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
# Message with placeholders
violation_message_template: "{action_type} blocked due to {category} violation. Please contact support."
```
**Supported Placeholders:**
- `{guardrail_name}`: Name of the guardrail (e.g. "panw-custom-message")
- `{category}`: Violation category (e.g. "malicious", "injection", "dlp")
- `{action_type}`: "Prompt" or "Response"
- `{default_message}`: The original technical error message
### Fail-Open Configuration
By default, the PANW guardrail operates in **fail-closed** mode for maximum security. If the PANW API is unavailable (timeout, rate limit, network error), requests are blocked. You can configure **fail-open** mode for high-availability scenarios where service continuity is critical.
docs/my-website/docs/proxy/logging.md
+58
View File
@@ -1827,6 +1827,64 @@ This approach allows you to:
- Share callbacks across different environments
- Version control callback files in cloud storage
#### Step 2c - Mounting Custom Callbacks in Helm/Kubernetes (Alternative)
When deploying with Helm or Kubernetes, you can mount custom callback Python files alongside your `config.yaml` using `subPath` to avoid overwriting the config directory.
**The Problem:**
Mounting a volume to a directory (e.g., `/app/`) would normally hide all existing files in that directory, including your `config.yaml`.
**The Solution:**
Use `subPath` in your `volumeMounts` to mount individual files without overwriting the entire directory.
**Example - Helm values.yaml:**
```yaml
# values.yaml
volumes:
- name: callback-files
configMap:
name: litellm-callback-files
volumeMounts:
- name: callback-files
mountPath: /app/custom_callbacks.py # Mount to specific FILE path
subPath: custom_callbacks.py # Required to avoid overwriting directory
```
**Create the ConfigMap with your callback file:**
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: litellm-callback-files
data:
custom_callbacks.py: |
from litellm.integrations.custom_logger import CustomLogger
class MyCustomHandler(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"Success! Model: {kwargs.get('model')}")
proxy_handler_instance = MyCustomHandler()
```
**Reference in your config.yaml:**
```yaml
litellm_settings:
callbacks: custom_callbacks.proxy_handler_instance
```
**How it works:**
1. The `subPath` parameter tells Kubernetes to mount only the specific file
2. This places `custom_callbacks.py` in `/app/` alongside your existing `config.yaml`
3. LiteLLM automatically finds the callback file in the same directory as the config
4. No files are overwritten or hidden
**Note:** You can mount multiple callback files by adding more `volumeMounts` entries, each with its own `subPath`.
#### Step 3 - Start proxy + test request
```shell
docs/my-website/docs/proxy/spend_logs_deletion.md
+12
View File
@@ -30,6 +30,9 @@ general_settings:
# Optional: set how frequently cleanup should run - default is daily
maximum_spend_logs_retention_interval: "1d" # Run cleanup daily
# Optional: set exact time for cleanup (Cron syntax)
maximum_spend_logs_cleanup_cron: "0 4 * * *" # Run at 04:00 AM daily
litellm_settings:
cache: true
cache_params:
@@ -51,6 +54,15 @@ How long logs should be kept before deletion. Supported formats:
How often the cleanup job should run. Uses the same format as above. If not set, cleanup will run every 24 hours if and only if `maximum_spend_logs_retention_period` is set.
#### `maximum_spend_logs_cleanup_cron` (optional)
Schedule the cleanup using standard cron syntax. This takes precedence over `maximum_spend_logs_retention_interval`.
Examples:
- `"0 4 * * *"` – Run at 04:00 AM daily
- `"0 0 * * 0"` – Run at midnight every Sunday
- `"*/30 * * * *"` – Run every 30 minutes
## How it works
### Step 1. Lock Acquisition (Optional with Redis)
docs/my-website/docs/routing.md
+4
View File
@@ -1333,6 +1333,10 @@ router = Router(model_list: Optional[list] = None,
cache_responses=True)
```
:::info
When configuring Redis caching in router settings, use `cache_kwargs` to pass additional Redis parameters, especially for non-string values that may fail when set via `REDIS_*` environment variables.
:::
## Pre-Call Checks (Context Window, EU-Regions)
Enable pre-call checks to filter out:
docs/my-website/docs/troubleshoot.md
+51 -3
View File
@@ -1,12 +1,60 @@
# Support & Talk with founders
# Troubleshooting & Support
## Information to Provide When Seeking Help
When reporting issues, please include as much of the following as possible. It's okay if you can't provide everything—especially in production scenarios where the trigger might be unknown. Sharing most of this information will help us assist you more effectively.
### 1. LiteLLM Configuration File
Your `config.yaml` file (redact sensitive info like API keys). Include number of workers if not in config.
### 2. Initialization Command
The command used to start LiteLLM (e.g., `litellm --config config.yaml --num_workers 8 --detailed_debug`).
### 3. LiteLLM Version
- Current version
- Version when the issue first appeared (if different)
- If upgraded, the version changed from → to
### 4. Environment Variables
Non-sensitive environment variables not in your config (e.g., `NUM_WORKERS`, `LITELLM_LOG`, `LITELLM_MODE`). Do not include passwords or API keys.
### 5. Server Specifications
CPU cores, RAM, OS, number of instances/replicas, etc.
### 6. Database and Redis Usage
- **Database:** Using database? (`DATABASE_URL` set), database type and version
- **Redis:** Using Redis? Redis version, configuration type (Standalone/Cluster/Sentinel).
### 7. Endpoints
The endpoint(s) you're using that are experiencing issues (e.g., `/chat/completions`, `/embeddings`).
### 8. Request Example
A realistic example of the request causing issues, including expected vs. actual response and any error messages.
### 9. Error Logs, Stack Traces, and Metrics
Full error logs, stack traces, and any images from service metrics (CPU, memory, request rates, etc.) that might help diagnose the issue.
---
## Support Channels
[Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
[Community Discord 💭](https://discord.gg/wuPM9dRgDw)
[Community Slack 💭](https://www.litellm.ai/support)
Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
[![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw)
[![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw)
docs/my-website/docs/troubleshoot/cpu_issues.md
+31
View File
@@ -0,0 +1,31 @@
# CPU Issue Classification & Reproduction
## 1. Classify the CPU Issue
Select the options that best describes the CPU behavior observed.
- [ ] CPU scales with traffic (RPS-driven)
- [ ] CPU increases without a traffic increase
- [ ] CPU increases after a LiteLLM upgrade
## 2. Can you reproduce the issue?
Before escalating, verify whether the CPU issue can be reproduced in a test environment that mirrors your production setup.
If reproducible, provide **detailed reproduction steps** along with any relevant requests or configuration used.
For guidance on the type of information we're looking for, see the [LiteLLM Troubleshooting Guide](../troubleshoot).
## 3. Issue Cannot Be Reproduced
If the CPU issue cannot be reproduced in a test environment that mirrors your production setup, please provide:
1. **Information from Section 1 and 2**
- CPU classification (Section 1)
- Reproduction attempts and environment details (Section 2)
2. **Additional context** to help investigate:
- **Workload:** A realistic sample of requests processed before and during the spike, including any recent configuration changes.
- **Metrics:** CPU usage, P50/P99 latency, memory usage. Please include **screenshots** of the metrics whenever possible.
- **Logs / Alerts:** Any relevant logs or alerts captured **before and during the spike**.
> Providing this information allows the team to analyze patterns, correlate spikes with traffic or configuration, and attempt to reproduce the issue internally. Without it, our engineers won't have enough information to look into the problem.
docs/my-website/docs/troubleshoot/memory_issues.md
+37
View File
@@ -0,0 +1,37 @@
# Memory Issue Classification & Reproduction
## 1. Classify the Memory Issue
Select the option(s) that best describe the memory behavior observed:
- [ ] Memory scales with traffic (RPS-driven)
- [ ] Memory increases without a traffic increase
- [ ] Memory increases after a LiteLLM upgrade
- [ ] Memory leak (memory continuously grows over time)
- [ ] Out of Memory (OOM) events or pod restarts
---
## 2. Can you reproduce the issue?
Before escalating, verify whether the memory or OOM issue can be reproduced in a test environment that mirrors your production deployment.
If reproducible, provide **detailed reproduction steps** along with any relevant requests, workloads, or configuration used.
For guidance on the type of information we’re looking for, see the [LiteLLM Troubleshooting Guide](../troubleshoot).
---
## 3. Issue Cannot Be Reproduced
If the memory or OOM issue cannot be reproduced in a test environment that mirrors production, please provide:
1. **Information from Sections 1 and 2**
- Memory/issue classification (Section 1)
- Reproduction attempts and environment details (Section 2)
2. **Additional context** to help investigate:
- **Workload:** A realistic sample of requests processed before and during the spike, including any recent configuration changes.
- **Metrics:** Memory usage, CPU usage, P50/P99 latency, and any pod restarts or OOM events. Please include **screenshots** of the metrics whenever possible.
- **Logs / Alerts:** Any relevant logs or alerts captured **before and during the spike**, including OOM errors or stack traces if available.
> Providing this information allows the team to analyze patterns, correlate memory spikes or OOMs with traffic or configuration, and attempt to reproduce the issue internally. Without it, our engineers will not have enough information to investigate the problem.
docs/my-website/docs/tutorials/claude_code_customer_tracking.md
+99
View File
@@ -0,0 +1,99 @@
# Claude Code - Granular Cost Tracking
Track Claude Code usage by customer or tags using LiteLLM proxy. This enables granular cost attribution for billing, budgeting, and analytics.
## How It Works
Claude Code supports custom headers via `ANTHROPIC_CUSTOM_HEADERS`. LiteLLM automatically tracks requests with specific headers for cost attribution.
## Tracking Options
Choose how you want to attribute costs:
| Track By | Header | Use Case |
|----------|--------|----------|
| Customer | `x-litellm-customer-id` | Bill customers, per-user budgets |
| Tags | `x-litellm-tags` | Project tracking, cost centers, environments |
## Environment Variables
| Variable | Description | Example |
|----------|-------------|---------|
| `ANTHROPIC_BASE_URL` | LiteLLM proxy URL | `http://localhost:4000` |
| `ANTHROPIC_API_KEY` | LiteLLM API key | `sk-1234` |
| `ANTHROPIC_CUSTOM_HEADERS` | Custom headers (`header-name: value` format) | See examples below |
## Option 1: Track by Customer
Use this to attribute costs to specific customers or end-users.
```bash
export ANTHROPIC_BASE_URL=http://localhost:4000
export ANTHROPIC_API_KEY=sk-1234
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-customer-id: claude-ishaan-local"
```
## Option 2: Track by Tags
Use this to attribute costs to projects, cost centers, or environments. Pass comma-separated tags.
```bash
export ANTHROPIC_BASE_URL=http://localhost:4000
export ANTHROPIC_API_KEY=sk-1234
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-tags: project:acme,env:prod,team:backend"
```
## Quick Start
### 1. Set Environment Variables
```bash
export ANTHROPIC_BASE_URL=http://localhost:4000
export ANTHROPIC_API_KEY=sk-1234
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-customer-id: claude-ishaan-local"
```
### 2. Use Claude Code
```bash
claude
```
All requests will now be tracked under the customer ID `claude-ishaan-local`.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/8f45872e-2d00-4d01-bf3d-4d6ae11d1396/ascreenshot_d2a745b8da4f4a56aaf2cac02871ef53_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/dd41eae3-2592-4bc9-a8d2-d6d02614cd2d/ascreenshot_43ec9ee48ad946cca49732f007e786fc_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/0c30309e-7117-4999-a3df-d22a2d5629c1/ascreenshot_d76a48c53b9a4fad8f6727baf4aa6a9c_text_export.jpeg)
### 3. View Usage in LiteLLM UI
Navigate to the **Logs** tab in the LiteLLM UI.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/ff774392-69f5-483e-83e2-fb749c94ee90/ascreenshot_d264fc04c9ee47edb047f61b6eb8c4d7_text_export.jpeg)
Click on a request to see details.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/5f71589b-5fdd-4759-9b6e-e6874be0eb21/ascreenshot_92dd86dadccb4764b1169c29c10dfe65_text_export.jpeg)
Filter by customer ID to see all requests for that customer.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-16/dd1c8aba-e75b-4714-9eee-c785e9db99af/ascreenshot_36aaec0fe12f4189b64f704a551e6729_text_export.jpeg)
## Supported Headers
| Header | Description |
|--------|-------------|
| `x-litellm-customer-id` | Track by customer/end-user ID |
| `x-litellm-end-user-id` | Alternative customer ID header |
| `x-litellm-tags` | Comma-separated tags for cost attribution |
## Related
- [Claude Code Quickstart](./claude_responses_api.md)
- [Customer Budgets](../proxy/customers.md)
- [Tag Budgets](../proxy/tag_budgets.md)
- [Track Usage for Coding Tools](./cost_tracking_coding.md)
docs/my-website/docs/tutorials/claude_code_websearch.md
+203
View File
@@ -0,0 +1,203 @@
import Image from '@theme/IdealImage';
# Claude Code - WebSearch Across All Providers
Enable Claude Code's web search tool to work with any provider (Bedrock, Azure, Vertex, etc.). LiteLLM automatically intercepts web search requests and executes them server-side.
<Image img={require('../../img/claude_code_websearch.png')} />
## Proxy Configuration
Add WebSearch interception to your `litellm_config.yaml`:
```yaml showLineNumbers title="litellm_config.yaml"
model_list:
- model_name: bedrock-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_region_name: us-east-1
# Enable WebSearch interception for providers
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- bedrock
- azure
- vertex_ai
search_tool_name: perplexity-search # Optional: specific search tool
# Configure search provider
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
## Quick Start
### 1. Configure LiteLLM Proxy
Create `config.yaml`:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: bedrock-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_region_name: us-east-1
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers: [bedrock]
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
### 2. Start Proxy
```bash showLineNumbers title="Start LiteLLM Proxy"
export PERPLEXITY_API_KEY=your-key
litellm --config config.yaml
```
### 3. Use with Claude Code
```bash showLineNumbers title="Configure Claude Code"
export ANTHROPIC_BASE_URL=http://localhost:4000
export ANTHROPIC_API_KEY=sk-1234
claude
```
Now use web search in Claude Code - it works with any provider!
## How It Works
When Claude Code sends a web search request, LiteLLM:
1. Intercepts the native `web_search` tool
2. Converts it to LiteLLM's standard format
3. Executes the search via Perplexity/Tavily
4. Returns the final answer to Claude Code
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM Proxy
participant B as Bedrock/Azure/etc
participant P as Perplexity/Tavily
CC->>LP: Request with web_search tool
Note over LP: Convert native tool<br/>to LiteLLM format
LP->>B: Request with converted tool
B-->>LP: Response: tool_use
Note over LP: Detect web search<br/>tool_use
LP->>P: Execute search
P-->>LP: Search results
LP->>B: Follow-up with results
B-->>LP: Final answer
LP-->>CC: Final answer with search results
```
**Result**: One API call from Claude Code → Complete answer with search results
## Supported Providers
| Provider | Native Web Search | With LiteLLM |
|----------|-------------------|--------------|
| **Anthropic** | ✅ Yes | ✅ Yes |
| **Bedrock** | ❌ No | ✅ Yes |
| **Azure** | ❌ No | ✅ Yes |
| **Vertex AI** | ❌ No | ✅ Yes |
| **Other Providers** | ❌ No | ✅ Yes |
## Search Providers
Configure which search provider to use. LiteLLM supports multiple search providers:
| Provider | `search_provider` Value | Environment Variable |
|----------|------------------------|----------------------|
| **Perplexity AI** | `perplexity` | `PERPLEXITYAI_API_KEY` |
| **Tavily** | `tavily` | `TAVILY_API_KEY` |
| **Exa AI** | `exa_ai` | `EXA_API_KEY` |
| **Parallel AI** | `parallel_ai` | `PARALLEL_AI_API_KEY` |
| **Google PSE** | `google_pse` | `GOOGLE_PSE_API_KEY`, `GOOGLE_PSE_ENGINE_ID` |
| **DataForSEO** | `dataforseo` | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` |
| **Firecrawl** | `firecrawl` | `FIRECRAWL_API_KEY` |
| **SearXNG** | `searxng` | `SEARXNG_API_BASE` (required) |
| **Linkup** | `linkup` | `LINKUP_API_KEY` |
See [all supported search providers](../search/index.md) for detailed setup instructions and provider-specific parameters.
## Configuration Options
### WebSearch Interception Parameters
| Parameter | Type | Required | Description | Example |
|-----------|------|----------|-------------|---------|
| `enabled_providers` | List[String] | Yes | List of providers to enable web search interception for | `[bedrock, azure, vertex_ai]` |
| `search_tool_name` | String | No | Specific search tool from `search_tools` config. If not set, uses first available search tool. | `perplexity-search` |
### Supported Provider Values
Use these values in `enabled_providers`:
| Provider | Value | Description |
|----------|-------|-------------|
| AWS Bedrock | `bedrock` | Amazon Bedrock Claude models |
| Azure OpenAI | `azure` | Azure-hosted models |
| Google Vertex AI | `vertex_ai` | Google Cloud Vertex AI |
| Any Other | Provider name | Any LiteLLM-supported provider |
### Complete Configuration Example
```yaml showLineNumbers title="Complete config.yaml"
model_list:
- model_name: bedrock-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_region_name: us-east-1
- model_name: azure-gpt4
litellm_params:
model: azure/gpt-4
api_base: https://my-azure.openai.azure.com
api_key: os.environ/AZURE_API_KEY
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- bedrock # Enable for AWS Bedrock
- azure # Enable for Azure OpenAI
- vertex_ai # Enable for Google Vertex
search_tool_name: perplexity-search # Optional: use specific search tool
# Configure search tools
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
- search_tool_name: tavily-search
litellm_params:
search_provider: tavily
api_key: os.environ/TAVILY_API_KEY
```
**How search tool selection works:**
- If `search_tool_name` is specified → Uses that specific search tool
- If `search_tool_name` is not specified → Uses first search tool in `search_tools` list
- In example above: Without `search_tool_name`, would use `perplexity-search` (first in list)
## Related
- [Claude Code Quickstart](./claude_responses_api.md)
- [Claude Code Cost Tracking](./claude_code_customer_tracking.md)
- [Using Non-Anthropic Models](./claude_non_anthropic_models.md)
docs/my-website/docs/tutorials/claude_mcp.md
+93
View File
@@ -0,0 +1,93 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Use Claude Code with MCPs
This tutorial shows how to connect MCP servers to Claude Code via LiteLLM Proxy.
Note: LiteLLM supports OAuth for MCP servers as well. [Learn more](https://docs.litellm.ai/docs/mcp#mcp-oauth)
## Connecting MCP Servers
You can also connect MCP servers to Claude Code via LiteLLM Proxy.
1. Add the MCP server to your `config.yaml`
<Tabs>
<TabItem value="github" label="GitHub MCP">
In this example, we'll add the Github MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
```
</TabItem>
<TabItem value="atlassian" label="Atlassian MCP">
In this example, we'll add the Atlassian MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
atlassian_mcp:
server_id: atlassian_mcp_id
url: "https://mcp.atlassian.com/v1/sse"
transport: "sse"
auth_type: oauth2
```
</TabItem>
</Tabs>
2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Use the MCP server in Claude Code
```bash
claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
```
For MCP servers that require dynamic client registration (such as Atlassian), please set `x-litellm-api-key: Bearer sk-LITELLM_VIRTUAL_KEY` instead of using `Authorization: Bearer LITELLM_VIRTUAL_KEY`.
4. Authenticate via Claude Code
a. Start Claude Code
```bash
claude
```
b. Authenticate via Claude Code
```bash
/mcp
```
c. Select the MCP server
```bash
> litellm_proxy
```
d. Start Oauth flow via Claude Code
```bash
> 1. Authenticate
2. Reconnect
3. Disable
```
e. Once completed, you should see this success message:
<img src={require('../../img/oauth_2_success.png').default} alt="OAuth 2.0 Success" style={{ width: '500px', height: 'auto' }} />
docs/my-website/docs/tutorials/claude_non_anthropic_models.md
+316
View File
@@ -0,0 +1,316 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Use Claude Code with Non-Anthropic Models
This tutorial shows how to use Claude Code with non-Anthropic models like OpenAI, Gemini, and other LLM providers through LiteLLM proxy.
:::info
LiteLLM automatically translates between different provider formats, allowing you to use any supported LLM provider with Claude Code while maintaining the Anthropic Messages API format.
:::
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers (OpenAI, Vertex AI, etc.)
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Configuration
### 1. Setup config.yaml
Create a configuration file with your preferred non-Anthropic models:
<Tabs>
<TabItem value="openai" label="OpenAI">
```yaml
model_list:
# OpenAI GPT-4o
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
# OpenAI GPT-4o-mini
- model_name: gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
```
Set your environment variables:
```bash
export OPENAI_API_KEY="your-openai-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
</TabItem>
<TabItem value="gemini" label="Google AI Studio">
```yaml
model_list:
# Google Gemini
- model_name: gemini-3.0-flash-exp
litellm_params:
model: gemini/gemini-3.0-flash-exp
api_key: os.environ/GEMINI_API_KEY
```
Set your environment variables:
```bash
export GEMINI_API_KEY="your-gemini-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
```yaml
model_list:
# Google Gemini
- model_name: vertex-gemini-3-flash-preview
litellm_params:
model: vertex_ai/gemini-3-flash-preview
vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json"
vertex_project: "my-test-project"
vertex_location: "us-east-1"
# Anthropic Claude
- model_name: anthropic-vertex
litellm_params:
model: vertex_ai/claude-3-sonnet@20240229
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json"
```
Set your environment variables:
```bash
export VERTEX_FILE_PATH_ENV_VAR="/path/to/service_account.json"
export LITELLM_MASTER_KEY="sk-1234567890"
```
</TabItem>
<TabItem value="multi" label="Azure OpenAI">
```yaml
model_list:
# Azure OpenAI
- model_name: azure-gpt-4
litellm_params:
model: azure/gpt-4
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
api_version: "2024-02-01"
```
Set your environment variables:
```bash
export AZURE_API_KEY="your-azure-api-key"
export AZURE_API_BASE="https://your-resource.openai.azure.com"
export LITELLM_MASTER_KEY="sk-1234567890"
```
</TabItem>
</Tabs>
### 2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Verify Setup
Test that your proxy is working correctly:
<Tabs>
<TabItem value="openai-test" label="OpenAI">
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
</TabItem>
<TabItem value="gemini-test" label="Google AI Studio">
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini-3.0-flash-exp",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
</TabItem>
<TabItem value="vertex-test" label="Vertex AI">
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini-3.0-flash-exp",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
</TabItem>
<TabItem value="azure-test" label="Azure OpenAI">
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-gpt-4",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
</TabItem>
</Tabs>
### 4. Configure Claude Code
Configure Claude Code to use your LiteLLM proxy:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
:::tip
The `LITELLM_MASTER_KEY` gives Claude Code access to all proxy models. You can also create virtual keys in the LiteLLM UI to limit access to specific models.
:::
### 5. Use Claude Code with Non-Anthropic Models
Start Claude Code and specify which model to use:
```bash
# Use OpenAI GPT-4o
claude --model gpt-4o
# Use OpenAI GPT-4o-mini for faster responses
claude --model gpt-4o-mini
# Use Google Gemini
claude --model gemini-3.0-flash-exp
# Use Vertex AI Gemini
claude --model vertex-gemini-3-flash-preview
# Use Vertex AI Anthropic Claude
claude --model anthropic-vertex
# Use Azure OpenAI
claude --model azure-gpt-4
```
## How It Works
LiteLLM acts as a unified interface that:
1. **Receives requests** from Claude Code in Anthropic Messages API format
2. **Translates** the request to the target provider's format (OpenAI, Gemini, etc.)
3. **Forwards** the request to the actual provider
4. **Translates** the response back to Anthropic Messages API format
5. **Returns** the response to Claude Code
This allows you to use Claude Code's interface with any LLM provider supported by LiteLLM.
## Advanced Features
### Load Balancing and Fallbacks
Configure multiple deployments with automatic fallback:
```yaml
model_list:
- model_name: gpt-4o # virtual model name
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o # same virtual name
litellm_params:
model: azure/gpt-4o
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
router_settings:
routing_strategy: simple-shuffle # Load balance between deployments
num_retries: 2
timeout: 30
```
### Usage Tracking and Budgets
Track usage and set budgets through the LiteLLM UI:
```yaml
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
database_url: "postgresql://..." # Enable database for tracking
general_settings:
store_model_in_db: true
```
Start the proxy with the UI:
```bash
litellm --config /path/to/config.yaml --detailed_debug
```
Access the UI at `http://0.0.0.0:4000/ui` to:
- View usage analytics
- Set budget limits per user/key
- Monitor costs across different providers
- Create virtual keys with specific permissions
## Supported Providers
LiteLLM supports 100+ providers. Here are some popular ones for use with Claude Code:
- **OpenAI**: GPT-4o, GPT-4o-mini, o1, o3-mini
- **Google**: Gemini 2.0 Flash, Gemini 1.5 Pro/Flash
- **Azure OpenAI**: All OpenAI models via Azure
- **AWS Bedrock**: Llama, Mistral, and other models
- **Vertex AI**: Gemini, Claude, and other models on Google Cloud
- **Groq**: Fast inference for Llama and Mixtral
- **Together AI**: Llama, Mixtral, and other open source models
- **Deepseek**: Deepseek-chat, Deepseek-coder
[View full list of supported providers →](https://docs.litellm.ai/docs/providers)
docs/my-website/docs/tutorials/claude_responses_api.md
+26 -114
View File
@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Claude Code
# Claude Code Quickstart
This tutorial shows how to call Claude models through LiteLLM proxy from Claude Code.
@@ -142,7 +142,7 @@ Common issues and solutions:
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
- Check LiteLLM logs for detailed error messages
## Using Multiple Models
## Using Bedrock/Vertex AI/Azure Foundry Models
Expand your configuration to support multiple providers and models:
@@ -151,25 +151,6 @@ Expand your configuration to support multiple providers and models:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
@@ -189,6 +170,24 @@ model_list:
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
# Azure Foundry
- model_name: claude-4-azure
litellm_params:
model: azure_ai/claude-opus-4-1
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE # https://my-resource.services.ai.azure.com/anthropic
# Google Vertex AI
- model_name: anthropic-vertex
litellm_params:
model: vertex_ai/claude-haiku-4-5@20251001
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json"
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
@@ -204,6 +203,12 @@ claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
# Use Azure Foundry deployment
claude --model claude-4-azure
# Use Vertex AI deployment
claude --model anthropic-vertex
```
</TabItem>
@@ -211,96 +216,3 @@ claude --model claude-bedrock
<Image img={require('../../img/release_notes/claude_code_demo.png')} style={{ width: '500px', height: 'auto' }} />
## Connecting MCP Servers
You can also connect MCP servers to Claude Code via LiteLLM Proxy.
:::note
Limitations:
- Currently, only HTTP MCP servers are supported
:::
1. Add the MCP server to your `config.yaml`
<Tabs>
<TabItem value="github" label="GitHub MCP">
In this example, we'll add the Github MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
```
</TabItem>
<TabItem value="atlassian" label="Atlassian MCP">
In this example, we'll add the Atlassian MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
atlassian_mcp:
server_id: atlassian_mcp_id
url: "https://mcp.atlassian.com/v1/sse"
transport: "sse"
auth_type: oauth2
```
</TabItem>
</Tabs>
2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Use the MCP server in Claude Code
```bash
claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
```
For MCP servers that require dynamic client registration (such as Atlassian), please set `x-litellm-api-key: Bearer sk-LITELLM_VIRTUAL_KEY` instead of using `Authorization: Bearer LITELLM_VIRTUAL_KEY`.
4. Authenticate via Claude Code
a. Start Claude Code
```bash
claude
```
b. Authenticate via Claude Code
```bash
/mcp
```
c. Select the MCP server
```bash
> litellm_proxy
```
d. Start Oauth flow via Claude Code
```bash
> 1. Authenticate
2. Reconnect
3. Disable
```
e. Once completed, you should see this success message:
<Image img={require('../../img/oauth_2_success.png')} style={{ width: '500px', height: 'auto' }} />
docs/my-website/docs/tutorials/cursor_integration.md
+30
View File
@@ -1,3 +1,5 @@
import Image from '@theme/IdealImage';
# Cursor Integration
Route Cursor IDE requests through LiteLLM for unified logging, budget controls, and access to any model.
@@ -76,6 +78,34 @@ Send a message. All requests now route through LiteLLM.
---
## Connecting MCP Servers
You can also connect MCP servers to Cursor via LiteLLM Proxy.
For official instructions on configuring MCP integration with Cursor, please refer to the Cursor documentation here: [https://cursor.com/en-US/docs/context/mcp](https://cursor.com/en-US/docs/context/mcp).
1. In Cursor Settings, go to the "Tools & MCP" tab and click "New MCP Server".
2. In your `mcp.json`, add the following configuration:
```
{
"mcpServers": {
"litellm": {
"url": "http://localhost:4000/everything/mcp",
"type": "http",
"headers": {
"Authorization": "Bearer sk-LITELLM_VIRTUAL_KEY"
}
}
}
}
```
3. LiteLLM's MCP will now appear under "Installed MCP Servers" in Cursor.
<Image img={require('../../img/cursor_mcp_installed.png')} />
## Troubleshooting
| Issue | Solution |
docs/my-website/img/claude_code_websearch.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.4 MiB

docs/my-website/img/cursor_mcp_installed.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 125 KiB

docs/my-website/img/mcp_playground.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 406 KiB

docs/my-website/img/mcp_tool_testing_playground.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 284 KiB

docs/my-website/img/release_notes/claude_code_websearch.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 MiB

docs/my-website/img/ui_deleted_keys_table.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 360 KiB

docs/my-website/img/ui_endpoint_activity.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 503 KiB

After

Width:  |  Height:  |  Size: 504 KiB

docs/my-website/release_notes/v1.80.15/index.md
+3 -3
View File
@@ -1,5 +1,5 @@
---
title: "[Preview] v1.80.15.rc.1 - Manus API Support"
title: "v1.80.15-stable - Manus API Support"
slug: "v1-80-15"
date: 2026-01-10T10:00:00
authors:
@@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.80.15.rc.1
docker.litellm.ai/berriai/litellm:v1.80.15-stable.1
```
</TabItem>
@@ -638,6 +638,6 @@ Users can now see Endpoint Activity Metrics in the UI.
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.14.rc.1)**
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.15-stable.1)**
docs/my-website/release_notes/v1.81.0/index.md
+517
View File
@@ -0,0 +1,517 @@
---
title: "v1.81.0 - Claude Code - Web Search Across All Providers"
slug: "v1-81-0"
date: 2026-01-18T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.0
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.0
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Claude Code** - Support for using web search across Bedrock, Vertex AI, and all LiteLLM providers
- **Major Change** - [50MB limit on image URL downloads](#major-change---chatcompletions-image-url-download-size-limit) to improve reliability
- **Performance** - [25% CPU Usage Reduction](#performance---25-cpu-usage-reduction) by removing premature model.dump() calls from the hot path
- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams.md) with spend and budget information at the time of deletion
---
## Claude Code - Web Search Across All Providers
<Image img={require('../../img/release_notes/claude_code_websearch.png')} />
This release brings web search support to Claude Code across all LiteLLM providers (Bedrock, Azure, Vertex AI, and more), enabling AI coding assistants to search the web for real-time information.
This means you can now use Claude Code's web search tool with any provider, not just Anthropic's native API. LiteLLM automatically intercepts web search requests and executes them server-side using your configured search provider (Perplexity, Tavily, Exa AI, and more).
Proxy Admins can configure web search interception in their LiteLLM proxy config to enable this capability for their teams using Claude Code with Bedrock, Azure, or any other supported provider.
[**Learn more →**](../../docs/tutorials/claude_code_websearch.md)
---
## Major Change - /chat/completions Image URL Download Size Limit
To improve reliability and prevent memory issues, LiteLLM now includes a configurable **50MB limit** on image URL downloads by default. Previously, there was no limit on image downloads, which could occasionally cause memory issues with very large images.
### How It Works
Requests with image URLs exceeding 50MB will receive a helpful error message:
```bash
curl -X POST 'https://your-litellm-proxy.com/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is in this image?"
},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/very-large-image.jpg"
}
}
]
}
]
}'
```
**Error Response:**
```json
{
"error": {
"message": "Error: Image size (75.50MB) exceeds maximum allowed size (50.0MB). url=https://example.com/very-large-image.jpg",
"type": "ImageFetchError"
}
}
```
### Configuring the Limit
The default 50MB limit works well for most use cases, but you can easily adjust it if needed:
**Increase the limit (e.g., to 100MB):**
```bash
export MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=100
```
**Disable image URL downloads (for security):**
```bash
export MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0
```
**Docker Configuration:**
```bash
docker run \
-e MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=100 \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.0
```
**Proxy Config (config.yaml):**
```yaml
general_settings:
master_key: sk-1234
# Set via environment variable
environment_variables:
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: "100"
```
### Why Add This?
This feature improves reliability by:
- Preventing memory issues from very large images
- Aligning with OpenAI's 50MB payload limit
- Validating image sizes early (when Content-Length header is available)
---
## Performance - 25% CPU Usage Reduction
LiteLLM now reduces CPU usage by removing premature `model.dump()` calls from the hot path in request processing. Previously, Pydantic model serialization was performed earlier and more frequently than necessary, causing unnecessary CPU overhead on every request. By deferring serialization until it is actually needed, LiteLLM reduces CPU usage and improves request throughput under high load.
---
## Deleted Keys Audit Table on UI
<Image img={require('../../img/ui_deleted_keys_table.png')} />
LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams.md).
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Features |
| -------- | ----- | -------- |
| OpenAI | `gpt-5.2-codex` | Code generation |
| Azure | `azure/gpt-5.2-codex` | Code generation |
| Cerebras | `cerebras/zai-glm-4.7` | Reasoning, function calling |
| Replicate | All chat models | Full support for all Replicate chat models |
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Add missing anthropic tool results in response - [PR #18945](https://github.com/BerriAI/litellm/pull/18945)
- Preserve web_fetch_tool_result in multi-turn conversations - [PR #18142](https://github.com/BerriAI/litellm/pull/18142)
- **[Gemini](../../docs/providers/gemini)**
- Add presence_penalty support for Google AI Studio - [PR #18154](https://github.com/BerriAI/litellm/pull/18154)
- Forward extra_headers in generateContent adapter - [PR #18935](https://github.com/BerriAI/litellm/pull/18935)
- Add medium value support for detail param - [PR #19187](https://github.com/BerriAI/litellm/pull/19187)
- **[Vertex AI](../../docs/providers/vertex)**
- Improve passthrough endpoint URL parsing and construction - [PR #17526](https://github.com/BerriAI/litellm/pull/17526)
- Add type object to tool schemas missing type field - [PR #19103](https://github.com/BerriAI/litellm/pull/19103)
- Keep type field in Gemini schema when properties is empty - [PR #18979](https://github.com/BerriAI/litellm/pull/18979)
- **[Bedrock](../../docs/providers/bedrock)**
- Add OpenAI-compatible service_tier parameter translation - [PR #18091](https://github.com/BerriAI/litellm/pull/18091)
- Add user auth in standard logging object for Bedrock passthrough - [PR #19140](https://github.com/BerriAI/litellm/pull/19140)
- Strip throughput tier suffixes from model names - [PR #19147](https://github.com/BerriAI/litellm/pull/19147)
- **[OCI](../../docs/providers/oci)**
- Handle OpenAI-style image_url object in multimodal messages - [PR #18272](https://github.com/BerriAI/litellm/pull/18272)
- **[Ollama](../../docs/providers/ollama)**
- Set finish_reason to tool_calls and remove broken capability check - [PR #18924](https://github.com/BerriAI/litellm/pull/18924)
- **[Watsonx](../../docs/providers/watsonx/index)**
- Allow passing scope ID for Watsonx inferencing - [PR #18959](https://github.com/BerriAI/litellm/pull/18959)
- **[Replicate](../../docs/providers/replicate)**
- Add all chat Replicate models support - [PR #18954](https://github.com/BerriAI/litellm/pull/18954)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add OpenRouter support for image/generation endpoints - [PR #19059](https://github.com/BerriAI/litellm/pull/19059)
- **[Volcengine](../../docs/providers/volcano)**
- Add max_tokens settings for Volcengine models (deepseek-v3-2, glm-4-7, kimi-k2-thinking) - [PR #19076](https://github.com/BerriAI/litellm/pull/19076)
- **Azure Model Router**
- New Model - Azure Model Router on LiteLLM AI Gateway - [PR #19054](https://github.com/BerriAI/litellm/pull/19054)
- **GPT-5 Models**
- Correct context window sizes for GPT-5 model variants - [PR #18928](https://github.com/BerriAI/litellm/pull/18928)
- Correct max_input_tokens for GPT-5 models - [PR #19056](https://github.com/BerriAI/litellm/pull/19056)
- **Text Completion**
- Support token IDs (list of integers) as prompt - [PR #18011](https://github.com/BerriAI/litellm/pull/18011)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Prevent dropping thinking when any message has thinking_blocks - [PR #18929](https://github.com/BerriAI/litellm/pull/18929)
- Fix anthropic token counter with thinking - [PR #19067](https://github.com/BerriAI/litellm/pull/19067)
- Add better error handling for Anthropic - [PR #18955](https://github.com/BerriAI/litellm/pull/18955)
- Fix Anthropic during call error - [PR #19060](https://github.com/BerriAI/litellm/pull/19060)
- **[Gemini](../../docs/providers/gemini)**
- Fix missing `completion_tokens_details` in Gemini 3 Flash when reasoning_effort is not used - [PR #18898](https://github.com/BerriAI/litellm/pull/18898)
- Fix Gemini Image Generation imageConfig parameters - [PR #18948](https://github.com/BerriAI/litellm/pull/18948)
- **[Vertex AI](../../docs/providers/vertex)**
- Fix Vertex AI 400 Error with CachedContent model mismatch - [PR #19193](https://github.com/BerriAI/litellm/pull/19193)
- Fix Vertex AI doesn't support structured output - [PR #19201](https://github.com/BerriAI/litellm/pull/19201)
- **[Bedrock](../../docs/providers/bedrock)**
- Fix Claude Code (`/messages`) Bedrock Invoke usage and request signing - [PR #19111](https://github.com/BerriAI/litellm/pull/19111)
- Fix model ID encoding for Bedrock passthrough - [PR #18944](https://github.com/BerriAI/litellm/pull/18944)
- Respect max_completion_tokens in thinking feature - [PR #18946](https://github.com/BerriAI/litellm/pull/18946)
- Fix header forwarding in Bedrock passthrough - [PR #19007](https://github.com/BerriAI/litellm/pull/19007)
- Fix Bedrock stability model usage issues - [PR #19199](https://github.com/BerriAI/litellm/pull/19199)
---
## LLM API Endpoints
#### Features
- **[/messages (Claude Code)](../../docs/providers/anthropic)**
- Add support for Tool Search on `/messages` API across Azure, Bedrock, and Anthropic API - [PR #19165](https://github.com/BerriAI/litellm/pull/19165)
- Track end-users with Claude Code (`/messages`) for better analytics and monitoring - [PR #19171](https://github.com/BerriAI/litellm/pull/19171)
- Add web search support using LiteLLM `/search` endpoint with Claude Code (`/messages`) - [PR #19263](https://github.com/BerriAI/litellm/pull/19263), [PR #19294](https://github.com/BerriAI/litellm/pull/19294)
- **[/messages (Claude Code) - Bedrock](../../docs/providers/bedrock)**
- Add support for Prompt Caching with Bedrock Converse on `/messages` - [PR #19123](https://github.com/BerriAI/litellm/pull/19123)
- Ensure budget tokens are passed to Bedrock Converse API correctly on `/messages` - [PR #19107](https://github.com/BerriAI/litellm/pull/19107)
- **[Responses API](../../docs/response_api)**
- Add support for caching for responses API - [PR #19068](https://github.com/BerriAI/litellm/pull/19068)
- Add retry policy support to responses API - [PR #19074](https://github.com/BerriAI/litellm/pull/19074)
- **Realtime API**
- Use non-streaming method for endpoint v1/a2a/message/send - [PR #19025](https://github.com/BerriAI/litellm/pull/19025)
- **Batch API**
- Fix batch deletion and retrieve - [PR #18340](https://github.com/BerriAI/litellm/pull/18340)
#### Bugs
- **General**
- Fix responses content can't be none - [PR #19064](https://github.com/BerriAI/litellm/pull/19064)
- Fix model name from query param in realtime request - [PR #19135](https://github.com/BerriAI/litellm/pull/19135)
- Fix video status/content credential injection for wildcard models - [PR #18854](https://github.com/BerriAI/litellm/pull/18854)
---
## Management Endpoints / UI
#### Features
**Virtual Keys**
- View deleted keys for audit purposes - [PR #18228](https://github.com/BerriAI/litellm/pull/18228), [PR #19268](https://github.com/BerriAI/litellm/pull/19268)
- Add status query parameter for keys list - [PR #19260](https://github.com/BerriAI/litellm/pull/19260)
- Refetch keys after key creation - [PR #18994](https://github.com/BerriAI/litellm/pull/18994)
- Refresh keys list on delete - [PR #19262](https://github.com/BerriAI/litellm/pull/19262)
- Simplify key generate permission error - [PR #18997](https://github.com/BerriAI/litellm/pull/18997)
- Add search to key edit team dropdown - [PR #19119](https://github.com/BerriAI/litellm/pull/19119)
**Teams & Organizations**
- View deleted teams for audit purposes - [PR #18228](https://github.com/BerriAI/litellm/pull/18228), [PR #19268](https://github.com/BerriAI/litellm/pull/19268)
- Add filters to organization table - [PR #18916](https://github.com/BerriAI/litellm/pull/18916)
- Add query parameters to `/organization/list` - [PR #18910](https://github.com/BerriAI/litellm/pull/18910)
- Add status query parameter for teams list - [PR #19260](https://github.com/BerriAI/litellm/pull/19260)
- Show internal users their spend only - [PR #19227](https://github.com/BerriAI/litellm/pull/19227)
- Allow preventing team admins from deleting members from teams - [PR #19128](https://github.com/BerriAI/litellm/pull/19128)
- Refactor team member icon buttons - [PR #19192](https://github.com/BerriAI/litellm/pull/19192)
**Models + Endpoints**
- Display health information in public model hub - [PR #19256](https://github.com/BerriAI/litellm/pull/19256), [PR #19258](https://github.com/BerriAI/litellm/pull/19258)
- Quality of life improvements for Anthropic models - [PR #19058](https://github.com/BerriAI/litellm/pull/19058)
- Create reusable model select component - [PR #19164](https://github.com/BerriAI/litellm/pull/19164)
- Edit settings model dropdown - [PR #19186](https://github.com/BerriAI/litellm/pull/19186)
- Fix model hub client side exception - [PR #19045](https://github.com/BerriAI/litellm/pull/19045)
**Usage & Analytics**
- Allow top virtual keys and models to show more entries - [PR #19050](https://github.com/BerriAI/litellm/pull/19050)
- Fix Y axis on model activity chart - [PR #19055](https://github.com/BerriAI/litellm/pull/19055)
- Add Team ID and Team Name in export report - [PR #19047](https://github.com/BerriAI/litellm/pull/19047)
- Add user metrics for Prometheus - [PR #18785](https://github.com/BerriAI/litellm/pull/18785)
**SSO & Auth**
- Allow setting custom MSFT Base URLs - [PR #18977](https://github.com/BerriAI/litellm/pull/18977)
- Allow overriding env var attribute names - [PR #18998](https://github.com/BerriAI/litellm/pull/18998)
- Fix SCIM GET /Users error and enforce SCIM 2.0 compliance - [PR #17420](https://github.com/BerriAI/litellm/pull/17420)
- Feature flag for SCIM compliance fix - [PR #18878](https://github.com/BerriAI/litellm/pull/18878)
**General UI**
- Add allowClear to dropdown components for better UX - [PR #18778](https://github.com/BerriAI/litellm/pull/18778)
- Add community engagement buttons - [PR #19114](https://github.com/BerriAI/litellm/pull/19114)
- UI Feedback Form - why LiteLLM - [PR #18999](https://github.com/BerriAI/litellm/pull/18999)
- Refactor user and team table filters to reusable component - [PR #19010](https://github.com/BerriAI/litellm/pull/19010)
- Adjusting new badges - [PR #19278](https://github.com/BerriAI/litellm/pull/19278)
#### Bugs
- Container API routes return 401 for non-admin users - routes missing from openai_routes - [PR #19115](https://github.com/BerriAI/litellm/pull/19115)
- Allow routing to regional endpoints for Containers API - [PR #19118](https://github.com/BerriAI/litellm/pull/19118)
- Fix Azure Storage circular reference error - [PR #19120](https://github.com/BerriAI/litellm/pull/19120)
- Fix prompt deletion fails with Prisma FieldNotFoundError - [PR #18966](https://github.com/BerriAI/litellm/pull/18966)
---
## AI Integrations
### Logging
- **[OpenTelemetry](../../docs/proxy/logging#opentelemetry)**
- Update semantic conventions to 1.38 (gen_ai attributes) - [PR #18793](https://github.com/BerriAI/litellm/pull/18793)
- **[LangSmith](../../docs/proxy/logging#langsmith)**
- Hoist thread grouping metadata (session_id, thread) - [PR #18982](https://github.com/BerriAI/litellm/pull/18982)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Include Langfuse logger in JSON logging when Langfuse callback is used - [PR #19162](https://github.com/BerriAI/litellm/pull/19162)
- **[Logfire](../../docs/observability/logfire)**
- Add ability to customize Logfire base URL through env var - [PR #19148](https://github.com/BerriAI/litellm/pull/19148)
- **General Logging**
- Enable JSON logging via configuration and add regression test - [PR #19037](https://github.com/BerriAI/litellm/pull/19037)
- Fix header forwarding for embeddings endpoint - [PR #18960](https://github.com/BerriAI/litellm/pull/18960)
- Preserve llm_provider-* headers in error responses - [PR #19020](https://github.com/BerriAI/litellm/pull/19020)
- Fix turn_off_message_logging not redacting request messages in proxy_server_request field - [PR #18897](https://github.com/BerriAI/litellm/pull/18897)
### Guardrails
- **[Grayswan](../../docs/proxy/guardrails/grayswan)**
- Implement fail-open option (default: True) - [PR #18266](https://github.com/BerriAI/litellm/pull/18266)
- **[Pangea](../../docs/proxy/guardrails/pangea)**
- Respect `default_on` during initialization - [PR #18912](https://github.com/BerriAI/litellm/pull/18912)
- **[Panw Prisma AIRS](../../docs/proxy/guardrails/panw_prisma_airs)**
- Add custom violation message support - [PR #19272](https://github.com/BerriAI/litellm/pull/19272)
- **General Guardrails**
- Fix SerializationIterator error and pass tools to guardrail - [PR #18932](https://github.com/BerriAI/litellm/pull/18932)
- Properly handle custom guardrails parameters - [PR #18978](https://github.com/BerriAI/litellm/pull/18978)
- Use clean error messages for blocked requests - [PR #19023](https://github.com/BerriAI/litellm/pull/19023)
- Guardrail moderation support with responses API - [PR #18957](https://github.com/BerriAI/litellm/pull/18957)
- Fix model-level guardrails not taking effect - [PR #18895](https://github.com/BerriAI/litellm/pull/18895)
---
## Spend Tracking, Budgets and Rate Limiting
- **Cost Calculation Fixes**
- Include IMAGE token count in cost calculation for Gemini models - [PR #18876](https://github.com/BerriAI/litellm/pull/18876)
- Fix negative text_tokens when using cache with images - [PR #18768](https://github.com/BerriAI/litellm/pull/18768)
- Fix image tokens spend logging for `/images/generations` - [PR #19009](https://github.com/BerriAI/litellm/pull/19009)
- Fix incorrect `prompt_tokens_details` in Gemini Image Generation - [PR #19070](https://github.com/BerriAI/litellm/pull/19070)
- Fix case-insensitive model cost map lookup - [PR #18208](https://github.com/BerriAI/litellm/pull/18208)
- **Pricing Updates**
- Correct pricing for `openrouter/openai/gpt-oss-20b` - [PR #18899](https://github.com/BerriAI/litellm/pull/18899)
- Add pricing for `azure_ai/claude-opus-4-5` - [PR #19003](https://github.com/BerriAI/litellm/pull/19003)
- Update Novita models prices - [PR #19005](https://github.com/BerriAI/litellm/pull/19005)
- Fix Azure Grok prices - [PR #19102](https://github.com/BerriAI/litellm/pull/19102)
- Fix GCP GLM-4.7 pricing - [PR #19172](https://github.com/BerriAI/litellm/pull/19172)
- Sync DeepSeek chat/reasoner to V3.2 pricing - [PR #18884](https://github.com/BerriAI/litellm/pull/18884)
- Correct cache_read pricing for gemini-2.5-pro models - [PR #18157](https://github.com/BerriAI/litellm/pull/18157)
- **Budget & Rate Limiting**
- Correct budget limit validation operator (>=) for team members - [PR #19207](https://github.com/BerriAI/litellm/pull/19207)
- Fix TPM 25% limiting by ensuring priority queue logic - [PR #19092](https://github.com/BerriAI/litellm/pull/19092)
- Cleanup spend logs cron verification, fix, and docs - [PR #19085](https://github.com/BerriAI/litellm/pull/19085)
---
## MCP Gateway
- Prevent duplicate MCP reload scheduler registration - [PR #18934](https://github.com/BerriAI/litellm/pull/18934)
- Forward MCP extra headers case-insensitively - [PR #18940](https://github.com/BerriAI/litellm/pull/18940)
- Fix MCP REST auth checks - [PR #19051](https://github.com/BerriAI/litellm/pull/19051)
- Fix generating two telemetry events in responses - [PR #18938](https://github.com/BerriAI/litellm/pull/18938)
- Fix MCP chat completions - [PR #19129](https://github.com/BerriAI/litellm/pull/19129)
---
## Performance / Loadbalancing / Reliability improvements
- **Performance Improvements**
- Remove bottleneck causing high CPU usage & overhead under heavy load - [PR #19049](https://github.com/BerriAI/litellm/pull/19049)
- Add CI enforcement for O(1) operations in `_get_model_cost_key` to prevent performance regressions - [PR #19052](https://github.com/BerriAI/litellm/pull/19052)
- Fix Azure embeddings JSON parsing to prevent connection leaks and ensure proper router cooldown - [PR #19167](https://github.com/BerriAI/litellm/pull/19167)
- Do not fallback to token counter if `disable_token_counter` is enabled - [PR #19041](https://github.com/BerriAI/litellm/pull/19041)
- **Reliability**
- Add fallback endpoints support - [PR #19185](https://github.com/BerriAI/litellm/pull/19185)
- Fix stream_timeout parameter functionality - [PR #19191](https://github.com/BerriAI/litellm/pull/19191)
- Fix model matching priority in configuration - [PR #19012](https://github.com/BerriAI/litellm/pull/19012)
- Fix num_retries in litellm_params as per config - [PR #18975](https://github.com/BerriAI/litellm/pull/18975)
- Handle exceptions without response parameter - [PR #18919](https://github.com/BerriAI/litellm/pull/18919)
- **Infrastructure**
- Add Custom CA certificates to boto3 clients - [PR #18942](https://github.com/BerriAI/litellm/pull/18942)
- Update boto3 to 1.40.15 and aioboto3 to 15.5.0 - [PR #19090](https://github.com/BerriAI/litellm/pull/19090)
- Make keepalive_timeout parameter work for Gunicorn - [PR #19087](https://github.com/BerriAI/litellm/pull/19087)
- **Helm Chart**
- Fix mount config.yaml as single file in Helm chart - [PR #19146](https://github.com/BerriAI/litellm/pull/19146)
- Sync Helm chart versioning with production standards and Docker versions - [PR #18868](https://github.com/BerriAI/litellm/pull/18868)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR |
| ----- | ----------- | ----------- | -- |
| `LiteLLM_ProxyModelTable` | New Columns | Added `created_at` and `updated_at` timestamp fields | [PR #18937](https://github.com/BerriAI/litellm/pull/18937) |
---
## Documentation Updates
- Add LiteLLM architecture md doc - [PR #19057](https://github.com/BerriAI/litellm/pull/19057), [PR #19252](https://github.com/BerriAI/litellm/pull/19252)
- Add troubleshooting guide - [PR #19096](https://github.com/BerriAI/litellm/pull/19096), [PR #19097](https://github.com/BerriAI/litellm/pull/19097), [PR #19099](https://github.com/BerriAI/litellm/pull/19099)
- Add structured issue reporting guides for CPU and memory issues - [PR #19117](https://github.com/BerriAI/litellm/pull/19117)
- Add Redis requirement warning for high-traffic deployments - [PR #18892](https://github.com/BerriAI/litellm/pull/18892)
- Update load balancing and routing with enable_pre_call_checks - [PR #18888](https://github.com/BerriAI/litellm/pull/18888)
- Updated pass_through with guided param - [PR #18886](https://github.com/BerriAI/litellm/pull/18886)
- Update message content types link and add content types table - [PR #18209](https://github.com/BerriAI/litellm/pull/18209)
- Add Redis initialization with kwargs - [PR #19183](https://github.com/BerriAI/litellm/pull/19183)
- Improve documentation for routing LLM calls via SAP Gen AI Hub - [PR #19166](https://github.com/BerriAI/litellm/pull/19166)
- Deleted Keys and Teams docs - [PR #19291](https://github.com/BerriAI/litellm/pull/19291)
- Claude Code end user tracking guide - [PR #19176](https://github.com/BerriAI/litellm/pull/19176)
- Add MCP troubleshooting guide - [PR #19122](https://github.com/BerriAI/litellm/pull/19122)
- Add auth message UI documentation - [PR #19063](https://github.com/BerriAI/litellm/pull/19063)
- Add guide for mounting custom callbacks in Helm/K8s - [PR #19136](https://github.com/BerriAI/litellm/pull/19136)
---
## Bug Fixes
- Fix Swagger UI path execute error with server_root_path in OpenAPI schema - [PR #18947](https://github.com/BerriAI/litellm/pull/18947)
- Normalize OpenAI SDK BaseModel choices/messages to avoid Pydantic serializer warnings - [PR #18972](https://github.com/BerriAI/litellm/pull/18972)
- Add contextual gap checks and word-form digits - [PR #18301](https://github.com/BerriAI/litellm/pull/18301)
- Clean up orphaned files from repository root - [PR #19150](https://github.com/BerriAI/litellm/pull/19150)
- Include proxy/prisma_migration.py in non-root - [PR #18971](https://github.com/BerriAI/litellm/pull/18971)
- Update prisma_migration.py - [PR #19083](https://github.com/BerriAI/litellm/pull/19083)
---
## New Contributors
* @yogeshwaran10 made their first contribution in [PR #18898](https://github.com/BerriAI/litellm/pull/18898)
* @theonlypal made their first contribution in [PR #18937](https://github.com/BerriAI/litellm/pull/18937)
* @jonmagic made their first contribution in [PR #18935](https://github.com/BerriAI/litellm/pull/18935)
* @houdataali made their first contribution in [PR #19025](https://github.com/BerriAI/litellm/pull/19025)
* @hummat made their first contribution in [PR #18972](https://github.com/BerriAI/litellm/pull/18972)
* @berkeyalciin made their first contribution in [PR #18966](https://github.com/BerriAI/litellm/pull/18966)
* @MateuszOssGit made their first contribution in [PR #18959](https://github.com/BerriAI/litellm/pull/18959)
* @xfan001 made their first contribution in [PR #18947](https://github.com/BerriAI/litellm/pull/18947)
* @nulone made their first contribution in [PR #18884](https://github.com/BerriAI/litellm/pull/18884)
* @debnil-mercor made their first contribution in [PR #18919](https://github.com/BerriAI/litellm/pull/18919)
* @hakhundov made their first contribution in [PR #17420](https://github.com/BerriAI/litellm/pull/17420)
* @rohanwinsor made their first contribution in [PR #19078](https://github.com/BerriAI/litellm/pull/19078)
* @pgolm made their first contribution in [PR #19020](https://github.com/BerriAI/litellm/pull/19020)
* @vikigenius made their first contribution in [PR #19148](https://github.com/BerriAI/litellm/pull/19148)
* @burnerburnerburnerman made their first contribution in [PR #19090](https://github.com/BerriAI/litellm/pull/19090)
* @yfge made their first contribution in [PR #19076](https://github.com/BerriAI/litellm/pull/19076)
* @danielnyari-seon made their first contribution in [PR #19083](https://github.com/BerriAI/litellm/pull/19083)
* @guilherme-segantini made their first contribution in [PR #19166](https://github.com/BerriAI/litellm/pull/19166)
* @jgreek made their first contribution in [PR #19147](https://github.com/BerriAI/litellm/pull/19147)
* @anand-kamble made their first contribution in [PR #19193](https://github.com/BerriAI/litellm/pull/19193)
* @neubig made their first contribution in [PR #19162](https://github.com/BerriAI/litellm/pull/19162)
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.15.rc.1...v1.81.0.rc.1)**
docs/my-website/sidebars.js
+46 -10
View File
@@ -108,15 +108,31 @@ const sidebars = {
{
type: "category",
label: "AI Tools (OpenWebUI, Claude Code, etc.)",
link: {
type: "generated-index",
title: "AI Tools",
description: "Integrate LiteLLM with AI tools like OpenWebUI, Claude Code, and more",
slug: "/ai_tools"
},
items: [
"tutorials/claude_responses_api",
"tutorials/openweb_ui",
{
type: "category",
label: "Claude Code",
items: [
"tutorials/claude_responses_api",
"tutorials/claude_code_customer_tracking",
"tutorials/claude_code_websearch",
"tutorials/claude_mcp",
"tutorials/claude_non_anthropic_models",
]
},
"tutorials/cost_tracking_coding",
"tutorials/cursor_integration",
"tutorials/github_copilot_integration",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
"tutorials/openai_codex",
"tutorials/openweb_ui"
"tutorials/openai_codex"
]
},
@@ -259,12 +275,21 @@ const sidebars = {
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI Usage Tracking",
items: [
"proxy/customer_usage",
"proxy/endpoint_activity"
]
},
{
type: "category",
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_logs_sessions"
"proxy/ui_logs_sessions",
"proxy/deleted_keys_teams"
]
}
],
@@ -314,7 +339,6 @@ const sidebars = {
"proxy/team_budgets",
"proxy/tag_budgets",
"proxy/customers",
"proxy/customer_usage",
"proxy/dynamic_rate_limit",
"proxy/rate_limit_tiers",
"proxy/temporary_budget_increase",
@@ -488,6 +512,7 @@ const sidebars = {
"mcp_control",
"mcp_cost",
"mcp_guardrail",
"mcp_troubleshoot",
]
},
"anthropic_unified",
@@ -602,6 +627,7 @@ const sidebars = {
label: "Azure AI",
items: [
"providers/azure_ai",
"providers/azure_ai/azure_model_router",
"providers/azure_ai_agents",
"providers/azure_ocr",
"providers/azure_document_intelligence",
@@ -691,6 +717,7 @@ const sidebars = {
"providers/galadriel",
"providers/github",
"providers/github_copilot",
"providers/chatgpt",
"providers/gradient_ai",
"providers/groq",
"providers/helicone",
@@ -841,6 +868,7 @@ const sidebars = {
"proxy/load_balancing",
"proxy/provider_budget_routing",
"proxy/reliability",
"proxy/fallback_management",
"proxy/tag_routing",
"proxy/timeout",
"wildcard_routing"
@@ -860,10 +888,11 @@ const sidebars = {
type: "category",
label: "Tutorials",
items: [
"tutorials/openweb_ui",
"tutorials/openai_codex",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
{
type: "link",
label: "AI Coding Tools (OpenWebUI, Claude Code, Gemini CLI, OpenAI Codex, etc.)",
href: "/docs/ai_tools",
},
"tutorials/anthropic_file_usage",
"tutorials/default_team_self_serve",
"tutorials/msft_sso",
@@ -873,7 +902,6 @@ const sidebars = {
"tutorials/presidio_pii_masking",
"tutorials/elasticsearch_logging",
"tutorials/gemini_realtime_with_audio",
"tutorials/claude_responses_api",
{
type: "category",
label: "LiteLLM Python SDK Tutorials",
@@ -970,6 +998,14 @@ const sidebars = {
],
},
"troubleshoot",
{
type: "category",
label: "Issue Reporting",
items: [
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
],
},
],
};
document.txt
-19
View File
@@ -1,19 +0,0 @@
LiteLLM provides a unified interface for calling 100+ different LLM providers.
Key capabilities:
- Translate requests to provider-specific formats
- Consistent OpenAI-compatible responses
- Retry and fallback logic across deployments
- Proxy server with authentication and rate limiting
- Support for streaming, function calling, and embeddings
Popular providers supported:
- OpenAI (GPT-4, GPT-3.5)
- Anthropic (Claude)
- AWS Bedrock
- Azure OpenAI
- Google Vertex AI
- Cohere
- And 95+ more
This allows developers to easily switch between providers without code changes.
enterprise/litellm_enterprise/proxy/__init__.py
+1
View File
@@ -0,0 +1 @@
# Package marker for enterprise proxy components.
enterprise/litellm_enterprise/proxy/common_utils/__init__.py
+1
View File
@@ -0,0 +1 @@
# Package marker for enterprise proxy common utilities.
enterprise/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.27"
version = "0.1.28"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.27"
version = "0.1.28"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",
flux2_test_image.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 172 KiB

litellm-proxy-extras/dist/litellm_proxy_extras-0.4.15-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.15.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.22-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.22.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.23-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.23.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120021_baseline_diff/migration.sql
-2
View File
@@ -1,2 +0,0 @@
-- This is an empty migration.
litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120539_baseline_diff/migration.sql
-2
View File
@@ -1,2 +0,0 @@
-- This is an empty migration.
litellm-proxy-extras/litellm_proxy_extras/migrations/20251219110931_add_deleted_keys_and_deleted_teams_tables/migration.sql
+117
View File
@@ -0,0 +1,117 @@
-- CreateTable
CREATE TABLE "LiteLLM_DeletedTeamTable" (
"id" TEXT NOT NULL,
"team_id" TEXT NOT NULL,
"team_alias" TEXT,
"organization_id" TEXT,
"object_permission_id" TEXT,
"admins" TEXT[],
"members" TEXT[],
"members_with_roles" JSONB NOT NULL DEFAULT '{}',
"metadata" JSONB NOT NULL DEFAULT '{}',
"max_budget" DOUBLE PRECISION,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
"models" TEXT[],
"max_parallel_requests" INTEGER,
"tpm_limit" BIGINT,
"rpm_limit" BIGINT,
"budget_duration" TEXT,
"budget_reset_at" TIMESTAMP(3),
"blocked" BOOLEAN NOT NULL DEFAULT false,
"model_spend" JSONB NOT NULL DEFAULT '{}',
"model_max_budget" JSONB NOT NULL DEFAULT '{}',
"team_member_permissions" TEXT[] DEFAULT ARRAY[]::TEXT[],
"model_id" INTEGER,
"created_at" TIMESTAMP(3),
"updated_at" TIMESTAMP(3),
"deleted_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"deleted_by" TEXT,
"deleted_by_api_key" TEXT,
"litellm_changed_by" TEXT,
CONSTRAINT "LiteLLM_DeletedTeamTable_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LiteLLM_DeletedVerificationToken" (
"id" TEXT NOT NULL,
"token" TEXT NOT NULL,
"key_name" TEXT,
"key_alias" TEXT,
"soft_budget_cooldown" BOOLEAN NOT NULL DEFAULT false,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
"expires" TIMESTAMP(3),
"models" TEXT[],
"aliases" JSONB NOT NULL DEFAULT '{}',
"config" JSONB NOT NULL DEFAULT '{}',
"user_id" TEXT,
"team_id" TEXT,
"permissions" JSONB NOT NULL DEFAULT '{}',
"max_parallel_requests" INTEGER,
"metadata" JSONB NOT NULL DEFAULT '{}',
"blocked" BOOLEAN,
"tpm_limit" BIGINT,
"rpm_limit" BIGINT,
"max_budget" DOUBLE PRECISION,
"budget_duration" TEXT,
"budget_reset_at" TIMESTAMP(3),
"allowed_cache_controls" TEXT[] DEFAULT ARRAY[]::TEXT[],
"allowed_routes" TEXT[] DEFAULT ARRAY[]::TEXT[],
"model_spend" JSONB NOT NULL DEFAULT '{}',
"model_max_budget" JSONB NOT NULL DEFAULT '{}',
"budget_id" TEXT,
"organization_id" TEXT,
"object_permission_id" TEXT,
"created_at" TIMESTAMP(3),
"created_by" TEXT,
"updated_at" TIMESTAMP(3),
"updated_by" TEXT,
"rotation_count" INTEGER DEFAULT 0,
"auto_rotate" BOOLEAN DEFAULT false,
"rotation_interval" TEXT,
"last_rotation_at" TIMESTAMP(3),
"key_rotation_at" TIMESTAMP(3),
"deleted_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"deleted_by" TEXT,
"deleted_by_api_key" TEXT,
"litellm_changed_by" TEXT,
CONSTRAINT "LiteLLM_DeletedVerificationToken_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedTeamTable_team_id_idx" ON "LiteLLM_DeletedTeamTable"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedTeamTable_deleted_at_idx" ON "LiteLLM_DeletedTeamTable"("deleted_at");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedTeamTable_organization_id_idx" ON "LiteLLM_DeletedTeamTable"("organization_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedTeamTable_team_alias_idx" ON "LiteLLM_DeletedTeamTable"("team_alias");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedTeamTable_created_at_idx" ON "LiteLLM_DeletedTeamTable"("created_at");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_token_idx" ON "LiteLLM_DeletedVerificationToken"("token");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_deleted_at_idx" ON "LiteLLM_DeletedVerificationToken"("deleted_at");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_user_id_idx" ON "LiteLLM_DeletedVerificationToken"("user_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_team_id_idx" ON "LiteLLM_DeletedVerificationToken"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_organization_id_idx" ON "LiteLLM_DeletedVerificationToken"("organization_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_key_alias_idx" ON "LiteLLM_DeletedVerificationToken"("key_alias");
-- CreateIndex
CREATE INDEX "LiteLLM_DeletedVerificationToken_created_at_idx" ON "LiteLLM_DeletedVerificationToken"("created_at");
litellm-proxy-extras/litellm_proxy_extras/migrations/20260116142756_update_deleted_keys_teams_table_routing_settings/migration.sql
+6
View File
@@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "router_settings" JSONB DEFAULT '{}';
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "router_settings" JSONB DEFAULT '{}';
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+99
View File
@@ -132,6 +132,49 @@ model LiteLLM_TeamTable {
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
// Audit table for deleted teams - preserves spend and team information for historical tracking
model LiteLLM_DeletedTeamTable {
id String @id @default(uuid())
team_id String // Original team_id
team_alias String?
organization_id String?
object_permission_id String?
admins String[]
members String[]
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the team
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([team_id])
@@index([deleted_at])
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
}
// Track spend, rate limit, budget Users
model LiteLLM_UserTable {
user_id String @id
@@ -259,6 +302,62 @@ model LiteLLM_VerificationToken {
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking
model LiteLLM_DeletedVerificationToken {
id String @id @default(uuid())
token String // Original token (hashed)
key_name String?
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
config Json @default("{}")
user_id String?
team_id String?
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
budget_id String?
organization_id String?
object_permission_id String?
created_at DateTime? // Original creation timestamp
created_by String? // Original creator
updated_at DateTime? // Last update timestamp before deletion
updated_by String? // Last user who updated before deletion
rotation_count Int? @default(0)
auto_rotate Boolean? @default(false)
rotation_interval String?
last_rotation_at DateTime?
key_rotation_at DateTime?
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the key
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([token])
@@index([deleted_at])
@@index([user_id])
@@index([team_id])
@@index([organization_id])
@@index([key_alias])
@@index([created_at])
}
model LiteLLM_EndUserTable {
user_id String @id
alias String? // admin-facing alias
litellm-proxy-extras/litellm_proxy_extras/utils.py
+227 -23
View File
@@ -11,6 +11,11 @@ from typing import Optional
from litellm_proxy_extras._logging import logger
try:
from litellm.caching.redis_cache import RedisCache
except ImportError:
RedisCache = None # type: ignore
def str_to_bool(value: Optional[str]) -> bool:
if value is None:
@@ -18,6 +23,154 @@ def str_to_bool(value: Optional[str]) -> bool:
return value.lower() in ("true", "1", "t", "y", "yes")
class MigrationLockManager:
"""Redis-based lock manager for database migrations.
Prevents concurrent Prisma migrations in multi-pod deployments by using
a distributed lock. Only one pod can hold the lock and run migrations at a time.
"""
MIGRATION_LOCK_KEY = "migration_lock"
LOCK_TTL_SECONDS = 300 # 5 minutes TTL
def __init__(self, redis_cache: Optional["RedisCache"] = None):
"""Initialize the migration lock manager.
Args:
redis_cache: Optional RedisCache instance for distributed locking.
If None, migrations run without lock protection (single instance mode).
"""
self.redis_cache = redis_cache
self.lock_acquired = False
self.pod_id = f"pod_{os.getpid()}_{int(time.time())}"
def _get_redis_lock_key(self) -> str:
"""Get Redis lock key for migration."""
return f"migration_lock:{self.MIGRATION_LOCK_KEY}"
def acquire_lock(self) -> bool:
"""Acquire migration lock using Redis SET NX.
Returns:
bool: True if lock acquired, False otherwise.
"""
if self.redis_cache is None:
logger.warning(
"Redis cache is not available, running migration without lock protection"
)
self.lock_acquired = True
return True
try:
lock_key = self._get_redis_lock_key()
# FIX: Use native Redis SET NX instead of RedisCache.set_cache()
# Original bug: set_cache() doesn't support nx parameter and returns None
# Fixed: Use redis_client.set() directly which returns True/False
acquired = self.redis_cache.redis_client.set(
name=lock_key,
value=self.pod_id,
nx=True, # Only set if key doesn't exist
ex=self.LOCK_TTL_SECONDS, # Set expiration time
)
if acquired:
self.lock_acquired = True
logger.info(f"Migration lock acquired by pod {self.pod_id}")
return True
else:
logger.info("Migration lock is already held by another pod")
return False
except Exception as e:
logger.warning(f"Failed to acquire migration lock: {e}")
return False
def wait_for_lock_release(
self, check_interval: int = 5, max_wait: int = 300
) -> bool:
"""Wait for another process to release the lock.
Args:
check_interval: Seconds to wait between lock acquisition attempts.
max_wait: Maximum seconds to wait for lock release.
Returns:
bool: True if lock acquired after waiting, False if timeout.
"""
if self.redis_cache is None:
logger.warning("Redis cache is not available, cannot wait for lock")
return False
logger.info(f"Waiting for migration lock to be released (max {max_wait}s)...")
start_time = time.time()
while time.time() - start_time < max_wait:
# Try to acquire lock using the public acquire_lock method
if self.acquire_lock():
logger.info(
f"Migration lock acquired after waiting by pod {self.pod_id}"
)
return True
time.sleep(check_interval)
logger.warning(f"Failed to acquire migration lock within {max_wait} seconds")
return False
def release_lock(self):
"""Release migration lock atomically using Lua script.
FIX: Use Lua script for atomic compare-and-delete to prevent race conditions.
Original bug: Non-atomic GET then DELETE allows another pod to acquire lock
between the GET and DELETE operations.
"""
if not self.lock_acquired or self.redis_cache is None:
return
try:
lock_key = self._get_redis_lock_key()
# FIX: Use Lua script for atomic compare-and-delete
# This prevents race condition where:
# 1. Pod A reads lock value (sees its own pod_id)
# 2. Lock TTL expires
# 3. Pod B acquires lock
# 4. Pod A deletes lock (deletes Pod B's lock!)
lua_script = """
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
else
return 0
end
"""
result = self.redis_cache.redis_client.eval(
lua_script,
1, # Number of keys
lock_key, # KEYS[1]
self.pod_id, # ARGV[1]
)
if result == 1:
logger.info(f"Migration lock released by pod {self.pod_id}")
else:
logger.warning(f"Pod {self.pod_id} cannot release lock (not owner)")
except Exception as e:
logger.warning(f"Failed to release migration lock: {e}")
finally:
self.lock_acquired = False
def __enter__(self):
"""Context manager entry - acquire lock when entering with statement."""
self.acquire_lock()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit - release lock when exiting with statement."""
self.release_lock()
def _get_prisma_env() -> dict:
"""Get environment variables for Prisma, handling offline mode if configured."""
@@ -25,7 +178,9 @@ def _get_prisma_env() -> dict:
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# These env vars prevent Prisma from attempting downloads
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
prisma_env["NPM_CONFIG_CACHE"] = os.getenv(
"NPM_CONFIG_CACHE", "/app/.cache/npm"
)
return prisma_env
@@ -34,29 +189,28 @@ def _get_prisma_command() -> str:
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# Primary location where Prisma Python package installs the CLI
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
# Check if custom path is provided (for flexibility)
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
if custom_cli_path and os.path.exists(custom_cli_path):
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
return custom_cli_path
# Check the default location
if os.path.exists(default_cli_path):
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
return default_cli_path
# If not found, log warning and fall back
logger.warning(
f"Prisma CLI not found at {default_cli_path}. "
"Falling back to Python wrapper (may attempt downloads)"
)
# Fall back to the Python wrapper (will work in online mode)
return "prisma"
class ProxyExtrasDBManager:
@staticmethod
def _get_prisma_dir() -> str:
@@ -119,7 +273,7 @@ class ProxyExtrasDBManager:
stdout=open(migration_file, "w"),
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
# 3. Mark the migration as applied since it represents current state
@@ -134,7 +288,7 @@ class ProxyExtrasDBManager:
],
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
return True
@@ -159,14 +313,20 @@ class ProxyExtrasDBManager:
@staticmethod
def _roll_back_migration(migration_name: str):
"""Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@@ -178,7 +338,7 @@ class ProxyExtrasDBManager:
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@@ -248,7 +408,7 @@ class ProxyExtrasDBManager:
if not database_url:
logger.error("DATABASE_URL not set")
return
diff_dir = (
Path(migrations_dir)
/ "migrations"
@@ -283,7 +443,7 @@ class ProxyExtrasDBManager:
check=True,
timeout=60,
stdout=f,
env=_get_prisma_env()
env=_get_prisma_env(),
)
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to generate migration diff: {e.stderr}")
@@ -313,7 +473,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
@@ -331,12 +491,18 @@ class ProxyExtrasDBManager:
try:
logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--applied",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e:
@@ -346,19 +512,57 @@ class ProxyExtrasDBManager:
)
@staticmethod
def setup_database(use_migrate: bool = False) -> bool:
def setup_database(
use_migrate: bool = False, redis_cache: Optional["RedisCache"] = None
) -> bool:
"""
Set up the database using either prisma migrate or prisma db push
Uses migrations from litellm-proxy-extras package
Set up the database using either prisma migrate or prisma db push.
Uses migrations from litellm-proxy-extras package.
In multi-instance environment, use redis lock to prevent concurrent execution.
Args:
schema_path (str): Path to the Prisma schema file
use_migrate (bool): Whether to use prisma migrate instead of db push
redis_cache: Redis cache instance for distributed locking
Returns:
bool: True if setup was successful, False otherwise
"""
schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
database_url = os.getenv("DATABASE_URL")
if not database_url:
logger.error("DATABASE_URL environment variable is not set")
return False
# Use MigrationLockManager to prevent concurrent migration execution
with MigrationLockManager(redis_cache) as lock_manager:
# Lock is already acquired in __enter__, check if it was successful
if not lock_manager.lock_acquired:
# Cannot acquire lock, another process is running migration
logger.info(
"Another pod is running migration, waiting for completion..."
)
# Wait for other process to complete migration
if not lock_manager.wait_for_lock_release():
logger.error("Failed to acquire migration lock after waiting")
return False
# Successfully acquired lock, proceed with migration
logger.info("Acquired migration lock, proceeding with migration")
return ProxyExtrasDBManager._execute_migration(use_migrate, schema_path)
@staticmethod
def _execute_migration(use_migrate: bool, schema_path: str) -> bool:
"""Execute the actual migration.
Args:
use_migrate: Whether to use prisma migrate instead of db push
schema_path: Path to the Prisma schema file
Returns:
bool: True if migration was successful, False otherwise
"""
for attempt in range(4):
original_dir = os.getcwd()
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
@@ -375,7 +579,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -413,7 +617,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.21"
version = "0.4.23"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.21"
version = "0.4.23"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+7 -1
View File
@@ -9,7 +9,7 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
warnings.filterwarnings(
"ignore", message=".*Accessing the.*attribute on the instance is deprecated.*"
)
### INIT VARIABLES ########################
### INIT VARIABLES #########################
import threading
import os
from typing import (
@@ -557,6 +557,7 @@ docker_model_runner_models: Set = set()
amazon_nova_models: Set = set()
stability_models: Set = set()
github_copilot_models: Set = set()
chatgpt_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
gigachat_models: Set = set()
@@ -812,6 +813,8 @@ def add_known_models():
stability_models.add(key)
elif value.get("litellm_provider") == "github_copilot":
github_copilot_models.add(key)
elif value.get("litellm_provider") == "chatgpt":
chatgpt_models.add(key)
elif value.get("litellm_provider") == "minimax":
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
@@ -1025,6 +1028,7 @@ models_by_provider: dict = {
"amazon_nova": amazon_nova_models,
"stability": stability_models,
"github_copilot": github_copilot_models,
"chatgpt": chatgpt_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
"gigachat": gigachat_models,
@@ -1458,6 +1462,8 @@ if TYPE_CHECKING:
from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig
from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig
from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig
from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig
from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig
litellm/_lazy_imports_registry.py
+4
View File
@@ -253,6 +253,8 @@ LLM_CONFIG_NAMES = (
"IBMWatsonXAudioTranscriptionConfig",
"GithubCopilotConfig",
"GithubCopilotResponsesAPIConfig",
"ChatGPTConfig",
"ChatGPTResponsesAPIConfig",
"ManusResponsesAPIConfig",
"GithubCopilotEmbeddingConfig",
"NebiusConfig",
@@ -648,6 +650,8 @@ _LLM_CONFIGS_IMPORT_MAP = {
"GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"),
"GithubCopilotResponsesAPIConfig": (".llms.github_copilot.responses.transformation", "GithubCopilotResponsesAPIConfig"),
"GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"),
"ChatGPTConfig": (".llms.chatgpt.chat.transformation", "ChatGPTConfig"),
"ChatGPTResponsesAPIConfig": (".llms.chatgpt.responses.transformation", "ChatGPTResponsesAPIConfig"),
"NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
"WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
"GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"),
litellm/_logging.py
+21 -1
View File
@@ -133,6 +133,26 @@ ALL_LOGGERS = [
]
def _get_loggers_to_initialize():
"""
Get all loggers that should be initialized with the JSON handler.
Includes third-party integration loggers (like langfuse) if they are
configured as callbacks.
"""
import litellm
loggers = list(ALL_LOGGERS)
# Add langfuse logger if langfuse is being used as a callback
langfuse_callbacks = {"langfuse", "langfuse_otel"}
all_callbacks = set(litellm.success_callback + litellm.failure_callback)
if langfuse_callbacks & all_callbacks:
loggers.append(logging.getLogger("langfuse"))
return loggers
def _initialize_loggers_with_handler(handler: logging.Handler):
"""
Initialize all loggers with a handler
@@ -140,7 +160,7 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
- Adds a handler to each logger
- Prevents bubbling to parent/root (critical to prevent duplicate JSON logs)
"""
for lg in ALL_LOGGERS:
for lg in _get_loggers_to_initialize():
lg.handlers.clear() # remove any existing handlers
lg.addHandler(handler) # add JSON formatter handler
lg.propagate = False # prevent bubbling to parent/root

Some files were not shown because too many files have changed in this diff Show More