tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-18 08:25:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into reuse-aiohttp-session-http-handler

Browse Source
This commit is contained in:
Dharamendra Kumar
2025-09-23 10:36:35 -07:00
parent 04acd4b739 8a012f9940
commit 34f51a20eb
221 changed files with 13600 additions and 3549 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+55 -2
View File
@@ -1050,6 +1050,51 @@ jobs:
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mapped_tests_coverage.xml
mv .coverage litellm_mapped_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mapped_tests_coverage.xml
- litellm_mapped_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
@@ -1779,8 +1824,8 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \
-e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e AZURE_API_BASE=$AZURE_API_BASE \
-e AZURE_API_VERSION="2024-05-01-preview" \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@@ -3175,6 +3220,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_enterprise_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests:
filters:
branches:
@@ -3219,6 +3270,7 @@ workflows:
- guardrails_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
@@ -3279,6 +3331,7 @@ workflows:
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
.github/workflows/test-mcp.yml
+48
View File
@@ -0,0 +1,48 @@
name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest==7.3.1"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install "pytest-cov==5.0.0"
poetry run pip install "pytest-asyncio==0.21.1"
poetry run pip install "respx==0.22.0"
poetry run pip install "pydantic==2.10.2"
poetry run pip install "mcp==1.10.1"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
- name: Run MCP tests
run: |
poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
Dockerfile
-3
View File
@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
README.md
+1 -1
View File
@@ -37,7 +37,7 @@ LiteLLM manages:
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
cookbook/litellm_proxy_server/cli_token_usage.py
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Example: Using CLI token with LiteLLM SDK
This example shows how to use the CLI authentication token
in your Python scripts after running `litellm-proxy login`.
"""
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"
def main():
"""Using CLI token with LiteLLM SDK"""
print("🚀 Using CLI Token with LiteLLM SDK")
print("=" * 40)
#litellm._turn_on_debug()
# Get the CLI token
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
return
print("✅ Found CLI token.")
available_models = litellm.get_valid_models(
check_provider_endpoint=True,
custom_llm_provider="litellm_proxy",
api_key=api_key,
api_base=LITELLM_BASE_URL
)
print("✅ Available models:")
if available_models:
for i, model in enumerate(available_models, 1):
print(f" {i:2d}. {model}")
else:
print(" No models available")
# Use with LiteLLM
try:
response = litellm.completion(
model="litellm_proxy/gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello from CLI token!"}],
api_key=api_key,
base_url=LITELLM_BASE_URL
)
print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
except Exception as e:
print(f"❌ Error: {e}")
if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
print("3. The token is stored locally at ~/.litellm/token.json")
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
+99 -35
View File
@@ -4,10 +4,10 @@ This document provides comprehensive instructions for AI agents to generate rele
## Required Inputs
1. **Release Version** (e.g., `v1.76.3-stable`)
1. **Release Version** (e.g., `v1.77.3-stable`)
2. **PR Diff/Changelog** - List of PRs with titles and contributors
3. **Previous Version Commit Hash** - To compare model pricing changes
4. **Reference Release Notes** - Previous release notes to follow style/format
4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
## Step-by-Step Process
@@ -26,12 +26,12 @@ git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
### 2. Release Notes Structure
Follow this exact structure based on `docs/my-website/release_notes/v1.76.1-stable/index.md`:
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable):
```markdown
---
title: "v1.76.X-stable - [Key Theme]"
slug: "v1-76-X"
title: "v1.77.X-stable - [Key Theme]"
slug: "v1-77-X"
date: YYYY-MM-DDTHH:mm:ss
authors: [standard author block]
hide_table_of_contents: false
@@ -43,23 +43,42 @@ hide_table_of_contents: false
## Key Highlights
[3-5 bullet points of major features]
## Major Changes
[Critical changes users need to know]
## Performance Improvements
[Performance-related changes]
## New Models / Updated Models
[Detailed model tables and provider updates]
#### New Model Support
[Model pricing table]
#### Features
[Provider-specific features organized by provider]
### Bug Fixes
[Provider-specific bug fixes organized by provider]
#### New Provider Support
[New provider integrations]
## LLM API Endpoints
[API-related features and fixes]
#### Features
[API-specific features organized by API type]
#### Bugs
[General bug fixes]
## Management Endpoints / UI
[Admin interface and management changes]
#### Features
[UI and management features]
#### Bugs
[Management-related bug fixes]
## Logging / Guardrail Integrations
[Observability and security features]
#### Features
[Organized by integration provider with proper doc links]
#### Guardrails
[Guardrail-specific features and fixes]
#### New Integration
[Major new integrations]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements]
@@ -86,21 +105,27 @@ hide_table_of_contents: false
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
- Group by provider
- Note pricing corrections
- Highlight deprecated models
- **Structure:**
- `#### New Model Support` - pricing table
- `#### Features` - organized by provider with documentation links
- `### Bug Fixes` - provider-specific bug fixes
- `#### New Provider Support` - major new provider integrations
- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
- Use bullet points under each provider for multiple features
- Separate features from bug fixes clearly
**Provider Features:**
- Group by provider (Gemini, OpenAI, Anthropic, etc.)
- Link to provider docs: `../../docs/providers/[provider_name]`
- Separate features from bug fixes
**API Endpoints:**
- Images API
- Video Generation (if applicable)
- Responses API
- Passthrough endpoints
- General chat completions
**LLM API Endpoints:**
- **Structure:**
- `#### Features` - organized by API type (Responses API, Batch API, etc.)
- `#### Bugs` - general bug fixes under **General** category
- **API Categories:**
- Responses API
- Batch API
- CountTokens API
- Images API
- Video Generation (if applicable)
- General (miscellaneous improvements)
- Use proper documentation links for each API type
**UI/Management:**
- Authentication changes
@@ -108,11 +133,19 @@ hide_table_of_contents: false
- Team management
- Key management
**Integrations:**
- Logging providers (Datadog, Braintrust, etc.)
- Guardrails
- Cost tracking
- Observability
**Logging / Guardrail Integrations:**
- **Structure:**
- `#### Features` - organized by integration provider with proper doc links
- `#### Guardrails` - guardrail-specific features and fixes
- `#### New Integration` - major new integrations
- **Integration Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- Other logging providers with proper doc links
- Use bullet points under each provider for multiple features
- Separate logging features from guardrails clearly
### 4. Documentation Linking Strategy
@@ -211,10 +244,41 @@ This release has a known issue...
:::
```
**Provider Features:**
**Provider Features (New Models / Updated Models section):**
```markdown
#### Features
- **[Provider Name](../../docs/providers/provider)**
- Feature description - [PR #XXXXX](link)
- Another feature description - [PR #YYYYY](link)
```
**API Features (LLM API Endpoints section):**
```markdown
#### Features
- **[API Name](../../docs/api_path)**
- Feature description - [PR #XXXXX](link)
- Another feature - [PR #YYYYY](link)
- **General**
- Miscellaneous improvements - [PR #ZZZZZ](link)
```
**Integration Features (Logging / Guardrail Integrations section):**
```markdown
#### Features
- **[Integration Name](../../docs/proxy/logging#integration)**
- Feature description - [PR #XXXXX](link)
- Bug fix description - [PR #YYYYY](link)
```
**Bug Fixes Pattern:**
```markdown
### Bug Fixes
- **[Provider/Component Name](../../docs/providers/provider)**
- Bug fix description - [PR #XXXXX](link)
```
### 10. Missing Documentation Check
docs/my-website/docs/completion/provider_specific_params.md
+54 -4
View File
@@ -423,7 +423,7 @@ model_list:
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
-d '{
"model": "llama-3-8b-instruct",
"messages": [
{
@@ -431,6 +431,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
"content": "What'\''s the weather like in Boston today?"
}
],
"adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
}'
```
"adapater_id": "my-special-adapter-id"
}'
```
## Provider-Specific Metadata Parameters
| Provider | Parameter | Use Case |
|----------|-----------|----------|
| **AWS Bedrock** | `requestMetadata` | Cost attribution, logging |
| **Gemini/Vertex AI** | `labels` | Resource labeling |
| **Anthropic** | `metadata` | User identification |
<Tabs>
<TabItem value="bedrock" label="AWS Bedrock">
```python
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
requestMetadata={"cost_center": "engineering"}
)
```
</TabItem>
<TabItem value="gemini" label="Gemini/Vertex AI">
```python
import litellm
response = litellm.completion(
model="vertex_ai/gemini-pro",
messages=[{"role": "user", "content": "Hello!"}],
labels={"environment": "production"}
)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
import litellm
response = litellm.completion(
model="anthropic/claude-3-sonnet-20240229",
messages=[{"role": "user", "content": "Hello!"}],
metadata={"user_id": "user123"}
)
```
</TabItem>
</Tabs>
docs/my-website/docs/completion/usage.md
+1
View File
@@ -26,6 +26,7 @@ response = completion(
print(response.usage)
```
> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
## Streaming Usage
docs/my-website/docs/enterprise.md
+5
View File
@@ -1,6 +1,11 @@
import Image from '@theme/IdealImage';
# Enterprise
:::info
✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
:::
For companies that need SSO, user management and professional support for LiteLLM Proxy
:::info
docs/my-website/docs/fine_tuning.md
+2
View File
@@ -13,6 +13,8 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
| Feature | Supported | Notes |
|-------|-------|-------|
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
| Logging | ✅ | Works across all logging integrations |
docs/my-website/docs/getting_started.md
+2 -1
View File
@@ -32,7 +32,8 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
More details 👉
- [Completion() function details](./completion/)
- [All supported models / providers on LiteLLM](./providers/)
- [Overview of supported models / providers on LiteLLM](./providers/)
- [Search all models / providers](https://models.litellm.ai/)
- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
## streaming
docs/my-website/docs/image_edits.md
+3
View File
@@ -18,6 +18,9 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
## Usage
### LiteLLM Python SDK
docs/my-website/docs/image_generation.md
+2
View File
@@ -279,6 +279,8 @@ print(f"response: {response}")
## Supported Providers
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Provider | Documentation Link |
|----------|-------------------|
| OpenAI | [OpenAI Image Generation →](./providers/openai) |
docs/my-website/docs/index.md
+9
View File
@@ -524,6 +524,15 @@ try:
except OpenAIError as e:
print(e)
```
### See How LiteLLM Transforms Your Requests
Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
You can try it out now directly on our Demo App!
Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack
docs/my-website/docs/mcp.md
-2
View File
@@ -114,7 +114,6 @@ mcp_servers:
description: "My custom MCP server"
auth_type: "api_key"
auth_value: "abc123"
spec_version: "2025-03-26"
```
**Configuration Options:**
@@ -716,7 +715,6 @@ mcp_servers:
url: https://mcp.deepwiki.com/mcp
transport: "http"
auth_type: "none"
spec_version: "2025-03-26"
access_groups: ["dev_group"]
```
docs/my-website/docs/moderation.md
+2
View File
@@ -130,6 +130,8 @@ Here's the exact json output and type you can expect from all moderation calls:
## **Supported Providers**
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Provider |
|-------------|
| OpenAI |
docs/my-website/docs/observability/callbacks.md
+17 -4
View File
@@ -5,13 +5,15 @@
liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
:::tip
**New to LiteLLM Callbacks?** Check out our comprehensive [Callback Management Guide](./callback_management.md) to understand when to use different callback hooks like `async_log_success_event` vs `async_post_call_success_hook`.
**New to LiteLLM Callbacks?**
- For proxy/server logging and observability, see the [Proxy Logging Guide](https://docs.litellm.ai/docs/proxy/logging).
- To write your own callback logic, see the [Custom Callbacks Guide](https://docs.litellm.ai/docs/observability/custom_callback).
:::
liteLLM supports:
- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
- [Callback Management Guide](./callback_management.md) - **Comprehensive guide for choosing the right hooks**
### Supported Callback Integrations
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [LangSmith](https://www.langchain.com/langsmith)
@@ -21,9 +23,20 @@ liteLLM supports:
- [Sentry](https://docs.sentry.io/platforms/python/)
- [PostHog](https://posthog.com/docs/libraries/python)
- [Slack](https://slack.dev/bolt-python/concepts)
- [Arize](https://docs.arize.com/)
- [PromptLayer](https://docs.promptlayer.com/)
This is **not** an extensive list. Please check the dropdown for all logging integrations.
### Related Cookbooks
Try out our cookbooks for code snippets and interactive demos:
- [Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Langfuse.ipynb)
- [Lunary Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Lunary.ipynb)
- [Arize Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Arize.ipynb)
- [Proxy + Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb)
- [PromptLayer Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_PromptLayer.ipynb)
### Quick Start
```python
docs/my-website/docs/observability/custom_callback.md
+17
View File
@@ -67,6 +67,23 @@ asyncio.run(completion())
- `async_post_call_success_hook` - Access user data + modify responses
- `async_pre_call_hook` - Modify requests before sending
### Example: Modifying the Response in async_post_call_success_hook
You can use `async_post_call_success_hook` to add custom headers or metadata to the response before it is returned to the client. For example:
```python
async def async_post_call_success_hook(data, user_api_key_dict, response):
# Add a custom header to the response
additional_headers = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {}
additional_headers["x-litellm-custom-header"] = "my-value"
if not hasattr(response, "_hidden_params"):
response._hidden_params = {}
response._hidden_params["additional_headers"] = additional_headers
return response
```
This allows you to inject custom metadata or headers into the response for downstream consumers. You can use this pattern to pass information to clients, proxies, or observability tools.
## Callback Functions
If you just want to log on a specific event (e.g. on input) - you can use callback functions.
docs/my-website/docs/providers/azure_ai_img_edit.md
+260
View File
@@ -0,0 +1,260 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Editing
Azure AI provides powerful image editing capabilities using FLUX models from Black Forest Labs to modify existing images based on text descriptions.
## Overview
| Property | Details |
|----------|---------|
| Description | Azure AI Image Editing uses FLUX models to modify existing images based on text prompts. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/edits`](#image-editing) |
## Setup
### API Key & Base URL & API Version
```python showLineNumbers
# Set your Azure AI API credentials
import os
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview" # Example API version
```
Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
## Supported Models
| Model Name | Description | Cost per Image |
|------------|-------------|----------------|
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding for editing | $0.04 |
## Image Editing
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic-edit" label="Basic Usage">
```python showLineNumbers title="Basic Image Editing"
import os
import base64
from pathlib import Path
import litellm
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
# Edit an image with a prompt
response = litellm.image_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add a winter theme with snow and cold colors",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"]
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="async-edit" label="Async Usage">
```python showLineNumbers title="Async Image Editing"
import os
import base64
from pathlib import Path
import litellm
import asyncio
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
async def edit_image():
# Edit image asynchronously
response = await litellm.aimage_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Make this image look like a watercolor painting",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"]
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("async_edited_image.png")
path.write_bytes(img_bytes)
# Run the async function
asyncio.run(edit_image())
```
</TabItem>
<TabItem value="advanced-edit" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Editing with Parameters"
import os
import base64
from pathlib import Path
import litellm
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
# Edit image with additional parameters
response = litellm.image_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add magical elements like floating crystals and mystical lighting",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"],
n=1
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("advanced_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Azure AI Image Editing Configuration"
model_list:
- model_name: azure-flux-kontext-edit
litellm_params:
model: azure_ai/FLUX.1-Kontext-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
api_version: os.environ/AZURE_AI_API_VERSION
model_info:
mode: image_edit
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make image editing requests with OpenAI Python SDK
<Tabs>
<TabItem value="openai-edit-sdk" label="OpenAI SDK">
```python showLineNumbers title="Azure AI Image Editing via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="sk-1234" # Your proxy API key
)
# Edit image with FLUX Kontext Pro
response = client.images.edit(
model="azure-flux-kontext-edit",
image=open("path/to/your/image.png", "rb"),
prompt="Transform this image into a beautiful oil painting style",
)
img_base64 = response.data[0].b64_json
img_bytes = base64.b64decode(img_base64)
path = Path("proxy_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="litellm-edit-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Azure AI Image Editing via Proxy - LiteLLM SDK"
import litellm
# Edit image through proxy
response = litellm.image_edit(
model="litellm_proxy/azure-flux-kontext-edit",
image=open("path/to/your/image.png", "rb"),
prompt="Add a mystical forest background with magical creatures",
api_base="http://localhost:4000",
api_key="sk-1234"
)
img_base64 = response.data[0].b64_json
img_bytes = base64.b64decode(img_base64)
path = Path("proxy_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="curl-edit" label="cURL">
```bash showLineNumbers title="Azure AI Image Editing via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/edits' \
--header 'Authorization: Bearer sk-1234' \
--form 'model="azure-flux-kontext-edit"' \
--form 'prompt="Convert this image to a vintage sepia tone with old-fashioned effects"' \
--form 'image=@"path/to/your/image.png"'
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Editing supports the following OpenAI-compatible parameters:
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `image` | file | The image file to edit | Required | File object or binary data |
| `prompt` | string | Text description of the desired changes | Required | `"Add snow and winter elements"` |
| `model` | string | The FLUX model to use for editing | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
| `n` | integer | Number of edited images to generate (You can specify only 1) | `1` | `1` |
| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
| `api_version` | string | API version for Azure AI | Required | `"2025-04-01-preview"` |
## Getting Started
1. Create an account at [Azure AI Studio](https://ai.azure.com/)
2. Deploy a FLUX model in your Azure AI Studio workspace
3. Get your API key and endpoint from the deployment details
4. Set your `AZURE_AI_API_KEY`, `AZURE_AI_API_BASE` and `AZURE_AI_API_VERSION` environment variables
5. Prepare your source image
6. Use `litellm.image_edit()` to modify your images with text instructions
## Additional Resources
- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)
docs/my-website/docs/providers/bedrock.md
+178
View File
@@ -308,6 +308,65 @@ print(response)
</TabItem>
</Tabs>
## Usage - Request Metadata
Attach metadata to Bedrock requests for logging and cost attribution.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
requestMetadata={
"cost_center": "engineering",
"user_id": "user123"
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on yaml**
```yaml
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
requestMetadata:
cost_center: "engineering"
```
**Set on request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="bedrock-claude-v1",
messages=[{"role": "user", "content": "Hello"}],
extra_body={
"requestMetadata": {"cost_center": "engineering"}
}
)
```
</TabItem>
</Tabs>
## Usage - Function Calling / Tool calling
LiteLLM supports tool calling via Bedrock's Converse and Invoke API's.
@@ -1822,6 +1881,59 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding
### API keys
This can be set as env variables or passed as **params to litellm.embedding()**
```python
import os
os.environ["AWS_ACCESS_KEY_ID"] = "" # Access key
os.environ["AWS_SECRET_ACCESS_KEY"] = "" # Secret access key
os.environ["AWS_REGION_NAME"] = "" # us-east-1, us-east-2, us-west-1, us-west-2
```
### Usage
```python
from litellm import embedding
response = embedding(
model="bedrock/amazon.titan-embed-text-v1",
input=["good morning from litellm"],
)
print(response)
```
#### Titan V2 - encoding_format support
```python
from litellm import embedding
# Float format (default)
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="float" # Returns float array
)
# Binary format
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="base64" # Returns base64 encoded binary
)
```
## Supported AWS Bedrock Embedding Models
| Model Name | Usage | Supported Additional OpenAI params |
|----------------------|---------------------------------------------|-----|
| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | `dimensions`, `encoding_format` |
| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)
### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)
## Image Generation
Use this for stable diffusion, and amazon nova canvas on bedrock
@@ -1901,6 +2013,39 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
### Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import image_generation
response = image_generation(
model="bedrock/amazon.nova-canvas-v1:0",
model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
prompt="A cute baby sea otter"
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: nova-canvas-inference-profile
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
aws_region_name: "eu-west-1"
```
</TabItem>
</Tabs>
## Supported AWS Bedrock Image Generation Models
| Model Name | Function Call |
@@ -2195,6 +2340,39 @@ response = completion(
Make the bedrock completion call
---
### Required AWS IAM Policy for AssumeRole
To use `aws_role_name` (STS AssumeRole) with LiteLLM, your IAM user or role **must** have permission to call `sts:AssumeRole` on the target role. If you see an error like:
```
An error occurred (AccessDenied) when calling the AssumeRole operation: User: arn:aws:sts::...:assumed-role/litellm-ecs-task-role/... is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::...:role/Enterprise/BedrockCrossAccountConsumer
```
This means the IAM identity running LiteLLM does **not** have permission to assume the target role. You must update your IAM policy to allow this action.
#### Example IAM Policy
Replace `<TARGET_ROLE_ARN>` with the ARN of the role you want to assume (e.g., `arn:aws:iam::123456789012:role/Enterprise/BedrockCrossAccountConsumer`).
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Resource": "<TARGET_ROLE_ARN>"
}
]
}
```
**Note:** The target role itself must also trust the calling IAM identity (via its trust policy) for AssumeRole to succeed. See [AWS AssumeRole docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-api.html) for more details.
---
<Tabs>
<TabItem value="sdk" label="SDK">
docs/my-website/docs/providers/vertex.md
+37 -169
View File
@@ -45,7 +45,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
## COMPLETION CALL
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@@ -69,7 +69,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@@ -189,14 +189,26 @@ print(json.loads(completion.choices[0].message.content))
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
```
or
```yaml
model_list:
- model_name: gemini-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
litellm_credential_name: vertex-global
vertex_project: project-name-here
vertex_location: global
base_model: gemini
model_info:
provider: Vertex
```
2. Start Proxy
```
@@ -210,7 +222,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@@ -262,7 +274,7 @@ except JSONSchemaValidationError as e:
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: "project-id"
@@ -283,7 +295,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@@ -391,7 +403,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="gemini-pro",
model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"googleSearch": {}}],
)
@@ -406,7 +418,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@@ -527,7 +539,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="gemini-pro",
model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"enterpriseWebSearch": {}}],
)
@@ -542,7 +554,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@@ -835,7 +847,7 @@ import litellm
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
response = litellm.completion(model="gemini-2.5-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
```
## Usage with LiteLLM Proxy Server
@@ -876,9 +888,9 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
vertex_location: "us-central1" # proj location
model_list:
-model_name: team1-gemini-pro
-model_name: team1-gemini-2.5-pro
litellm_params:
model: gemini-pro
model: gemini-2.5-pro
```
</TabItem>
@@ -905,7 +917,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
)
response = client.chat.completions.create(
model="team1-gemini-pro",
model="team1-gemini-2.5-pro",
messages = [
{
"role": "user",
@@ -925,7 +937,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "team1-gemini-pro",
"model": "team1-gemini-2.5-pro",
"messages": [
{
"role": "user",
@@ -975,7 +987,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json,
vertex_project="my-special-project",
@@ -1039,7 +1051,7 @@ In certain use-cases you may need to make calls to the models and pass [safety s
```python
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
safety_settings=[
{
@@ -1153,7 +1165,7 @@ litellm.vertex_ai_safety_settings = [
},
]
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
)
```
@@ -1212,7 +1224,7 @@ litellm.vertex_location = "us-central1 # Your Location
## Gemini Pro
| Model Name | Function Call |
|------------------|--------------------------------------|
| gemini-pro | `completion('gemini-pro', messages)`, `completion('vertex_ai/gemini-pro', messages)` |
| gemini-2.5-pro | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` |
## Fine-tuned Models
@@ -1307,7 +1319,7 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
## Gemini Pro Vision
| Model Name | Function Call |
|------------------|--------------------------------------|
| gemini-pro-vision | `completion('gemini-pro-vision', messages)`, `completion('vertex_ai/gemini-pro-vision', messages)`|
| gemini-2.5-pro-vision | `completion('gemini-2.5-pro-vision', messages)`, `completion('vertex_ai/gemini-2.5-pro-vision', messages)`|
## Gemini 1.5 Pro (and Vision)
| Model Name | Function Call |
@@ -1321,7 +1333,7 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
#### Using Gemini Pro Vision
Call `gemini-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
Call `gemini-2.5-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
LiteLLM Supports the following image types passed in `url`
- Images with Cloud Storage URIs - gs://cloud-samples-data/generative-ai/image/boats.jpeg
@@ -1339,7 +1351,7 @@ LiteLLM Supports the following image types passed in `url`
import litellm
response = litellm.completion(
model = "vertex_ai/gemini-pro-vision",
model = "vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@@ -1377,7 +1389,7 @@ image_path = "cached_logo.jpg"
# Getting the base64 string
base64_image = encode_image(image_path)
response = litellm.completion(
model="vertex_ai/gemini-pro-vision",
model="vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@@ -1433,7 +1445,7 @@ tools = [
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
model="vertex_ai/gemini-pro-vision",
model="vertex_ai/gemini-2.5-pro-vision",
messages=messages,
tools=tools,
)
@@ -2509,150 +2521,6 @@ print("response from proxy", response)
</TabItem>
</Tabs>
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME = "litellm-testing-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-test-project"
```
### Usage
#### 1. Create a file of batch requests for vertex
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**
Each `body` in the file should be an **OpenAI API request**
Create a file called `vertex_batch_completions.jsonl` in the current working directory, the `model` should be the Vertex AI model name
```
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload a File of batch requests
For `vertex_ai` litellm will upload the file to the provided `GCS_BUCKET_NAME`
```python
import os
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_name = "vertex_batch_completions.jsonl" #
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)
file_obj = oai_client.files.create(
file=open(file_path, "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use vertex_ai for this file upload
)
```
**Expected Response**
```json
{
"id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"bytes": 416,
"created_at": 1733392026,
"filename": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"status_details": null
}
```
#### 3. Create a batch
```python
batch_input_file_id = file_obj.id # use `file_obj` from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # example input_file_id = "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/c2b1b785-252b-448c-b180-033c4c63b3ce"
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1733392026,
"endpoint": "",
"input_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
#### 4. Retrieve a batch
```python
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id,
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1736500100,
"endpoint": "",
"input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
## **Fine Tuning APIs**
docs/my-website/docs/providers/vertex_batch.md
+264
View File
@@ -0,0 +1,264 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME="my-batch-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-project"
```
### Usage
Follow this complete workflow: create JSONL file → upload file → create batch → retrieve batch status → get file content
#### 1. Create a JSONL file of batch requests
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**.
Each `body` in the file should be an **OpenAI API request**.
Create a file called `batch_requests.jsonl` with your requests:
```jsonl
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload the file
Upload your JSONL file. For `vertex_ai`, the file will be stored in your configured GCS bucket provided by `GCS_BUCKET_NAME`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="upload_file.py"
from openai import OpenAI
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_obj = oai_client.files.create(
file=open("batch_requests.jsonl", "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"File uploaded with ID: {file_obj.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Upload File"
curl --request POST \
--url http://localhost:4000/v1/files \
--header 'Content-Type: multipart/form-data' \
--form purpose=batch \
--form file=@batch_requests.jsonl \
--form custom_llm_provider=vertex_ai
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"bytes": 416,
"created_at": 1758303684,
"filename": "litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"expires_at": null,
"status_details": null
}
```
#### 3. Create a batch
Create a batch job using the uploaded file ID.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="create_batch.py"
batch_input_file_id = file_obj.id # from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch created with ID: {create_batch_response.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Create Batch Request"
curl --request POST \
--url http://localhost:4000/v1/batches \
--header 'Content-Type: application/json' \
--data '{
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"custom_llm_provider": "vertex_ai"
}'
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite",
"request_counts": null,
"usage": null
}
```
#### 4. Retrieve batch status
Check the status of your batch job. The batch will progress through states: `validating` → `in_progress` → `completed`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="retrieve_batch.py"
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch status: {retrieved_batch.status}")
if retrieved_batch.status == "completed":
print(f"Output file: {retrieved_batch.output_file_id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Retrieve Batch Status"
curl --request GET \
--url 'http://localhost:4000/batches/7814463557919047680?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response (when completed):**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/prediction-model-2025-09-19T21:26:51.569037Z/predictions.jsonl",
"request_counts": null,
"usage": null
}
```
#### 5. Get file content
Once the batch is completed, retrieve the results using the `output_file_id` from the batch response.
**Important:** The `output_file_id` must be URL encoded when used in the request path.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="get_file_content.py"
import urllib.parse
import json
output_file_id = retrieved_batch.output_file_id
# URL encode the file ID
encoded_file_id = urllib.parse.quote_plus(output_file_id)
# Get file content
file_content = oai_client.files.content(
file_id=encoded_file_id,
extra_body={"custom_llm_provider": "vertex_ai"}
)
# Process the results
for line in file_content.text.strip().split('\n'):
result = json.loads(line)
print(f"Request: {result['request']}")
print(f"Response: {result['response']}")
print("---")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Get File Content"
# Note: The file ID must be URL encoded
curl --request GET \
--url 'http://localhost:4000/files/gs%253A%252F%252Fmy-batch-bucket%252Flitellm-vertex-files%252Fpublishers%252Fgoogle%252Fmodels%252Fgemini-2.5-flash-lite%252Fprediction-model-2025-09-19T21%253A26%253A51.569037Z%252Fpredictions.jsonl/content?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response:**
The response contains JSONL format with one result per line:
```jsonl
{"status":"","processed_time":"2025-09-19T21:29:47.352+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are a helpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.48079710006713866,"content":{"parts":[{"text":"Hello there! It's nice to meet you"}],"role":"model"},"finishReason":"MAX_TOKENS"}],"createTime":"2025-09-19T21:29:47.484619Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaIvKHdvshMIP_aOtuAg","usageMetadata":{"candidatesTokenCount":10,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":10}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":19,"trafficType":"ON_DEMAND"}}}
{"status":"","processed_time":"2025-09-19T21:29:47.358+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are an unhelpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.6168075137668185,"content":{"parts":[{"text":"I am unable to assist with this request."}],"role":"model"},"finishReason":"STOP"}],"createTime":"2025-09-19T21:29:47.470889Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaOneHISShMIP28nA8QQ","usageMetadata":{"candidatesTokenCount":9,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":9}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":18,"trafficType":"ON_DEMAND"}}}
```
docs/my-website/docs/proxy/budget_reset_and_tz.md
+1
View File
@@ -29,5 +29,6 @@ Common timezone values:
- `US/Pacific` - Pacific Time
- `Europe/London` - UK Time
- `Asia/Kolkata` - Indian Standard Time (IST)
- `Asia/Bangkok` - Indochina Time (ICT)
- `Asia/Tokyo` - Japan Standard Time
- `Australia/Sydney` - Australian Eastern Time
docs/my-website/docs/proxy/caching.md
+14
View File
@@ -958,6 +958,19 @@ curl http://localhost:4000/v1/chat/completions \
</Tabs>
## Redis max_connections
You can set the `max_connections` parameter in your `cache_params` for Redis. This is passed directly to the Redis client and controls the maximum number of simultaneous connections in the pool. If you see errors like `No connection available`, try increasing this value:
```yaml
litellm_settings:
cache: true
cache_params:
type: redis
max_connections: 100
```
## Supported `cache_params` on proxy config.yaml
```yaml
@@ -966,6 +979,7 @@ cache_params:
ttl: Optional[float]
default_in_memory_ttl: Optional[float]
default_in_redis_ttl: Optional[float]
max_connections: Optional[Int]
# Type of cache (options: "local", "redis", "s3")
type: s3
docs/my-website/docs/proxy/config_settings.md
+1
View File
@@ -50,6 +50,7 @@ litellm_settings:
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
namespace: "litellm.caching.caching" # namespace for redis cache
max_connections: 100 # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py.
# Optional - Redis Cluster Settings
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
docs/my-website/docs/proxy/custom_sso.md
+1 -3
View File
@@ -1,9 +1,7 @@
# ✨ Event Hooks for SSO Login
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
:::
## Overview
docs/my-website/docs/proxy/db_deadlocks.md
+26
View File
@@ -84,3 +84,29 @@ LiteLLM emits the following prometheus metrics to monitor the health/status of t
| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory |
| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis |
## Troubleshooting: Redis Connection Errors
You may see errors like:
```
LiteLLM Redis Caching: async async_increment() - Got exception from REDIS No connection available., Writing value=21
LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS No connection available., Writing value=None
```
This means all available Redis connections are in use, and LiteLLM cannot obtain a new connection from the pool. This can happen under high load or with many concurrent proxy requests.
**Solution:**
- Increase the `max_connections` parameter in your Redis config section in `proxy_config.yaml` to allow more simultaneous connections. For example:
```yaml
litellm_settings:
cache: True
cache_params:
type: redis
max_connections: 100 # Increase as needed for your traffic
```
Adjust this value based on your expected concurrency and Redis server capacity.
docs/my-website/docs/proxy/deploy.md
+1
View File
@@ -13,6 +13,7 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
docs/my-website/docs/proxy/dynamic_rate_limit.md
+241
View File
@@ -0,0 +1,241 @@
# Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
## Quick Start Usage
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python showLineNumbers title="test.py"
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
## [BETA] Set Priority / Reserve Quota
Reserve TPM/RPM capacity for different environments or use cases. This ensures critical production workloads always have guaranteed capacity, while development or lower-priority tasks use remaining quota.
**Use Cases:**
- Production vs Development environments
- Real-time applications vs batch processing
- Critical services vs experimental features
:::tip
Reserving TPM/RPM on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
### How Priority Reservation Works
Priority reservation allocates a percentage of your model's total TPM/RPM to specific priority levels. Keys with higher priority get guaranteed access to their reserved quota first.
**Example Scenario:**
- Model has 10 RPM total capacity
- Priority reservation: `{"prod": 0.9, "dev": 0.1}`
- Result: Production keys get 9 RPM guaranteed, Development keys get 1 RPM guaranteed
### Configuration
#### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 10 # Total model capacity
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
priority_reservation:
"prod": 0.9 # 90% reserved for production (9 RPM)
"dev": 0.1 # 10% reserved for development (1 RPM)
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your.env
```
**Configuration Details:**
`priority_reservation`: Dict[str, float]
- **Key (str)**: Priority level name (can be any string like "prod", "dev", "critical", etc.)
- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
- **Note**: Values should sum to 1.0 or less
**Start Proxy**
```bash
litellm --config /path/to/config.yaml
```
#### 2. Create Keys with Priority Levels
**Production Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "prod"}
}'
```
**Development Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "dev"}
}'
```
**Expected Response for both:**
```json
{
"key": "sk-...",
"metadata": {"priority": "prod"}, // or "dev"
...
}
```
#### 3. Test Priority Allocation
**Test Production Key (should get 9 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-prod-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from prod"}]
}'
```
**Test Development Key (should get 1 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-dev-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from dev"}]
}'
```
### Expected Behavior
With the configuration above:
1. **Production keys** can make up to 9 requests per minute
2. **Development keys** can make up to 1 request per minute
3. Production requests are never blocked by development usage
**Rate Limit Error Example:**
```json
{
"error": {
"message": "Key=sk-dev-... over available RPM=0. Model RPM=10, Reserved RPM for priority 'dev'=1, Active keys=1",
"type": "rate_limit_exceeded",
"code": 429
}
}
```
### Demo Video
This video walks through setting up dynamic rate limiting with priority reservation and locust tests to validate the behavior.
<iframe width="840" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7
" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
docs/my-website/docs/proxy/guardrails/bedrock.md
+4
View File
@@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem';
# Bedrock Guardrails
:::tip ⚡️
If you haven't set up or authenticated your Bedrock provider yet, see the [Bedrock Provider Setup & Authentication Guide](../../providers/bedrock.md).
:::
LiteLLM supports Bedrock guardrails via the [Bedrock ApplyGuardrail API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html).
## Quick Start
docs/my-website/docs/proxy/load_balancing.md
+3
View File
@@ -172,6 +172,9 @@ router_settings:
redis_host: <your redis host>
redis_password: <your redis password>
redis_port: 1992
cache_params:
type: redis
max_connections: 100 # maximum Redis connections in the pool; tune based on expected concurrency/load
```
## Router settings on config - routing_strategy, model_group_alias
docs/my-website/docs/proxy/self_serve.md
+5 -1
View File
@@ -227,7 +227,7 @@ export PROXY_LOGOUT_URL="https://www.google.com"
<Image img={require('../../img/ui_logout.png')} style={{ width: '400px', height: 'auto' }} />
### Set max budget for internal users
### Set default max budget for internal users
Automatically apply budget per internal user when they sign up. By default the table will be checked every 10 minutes, for users to reset. To modify this, [see this](./users.md#reset-budgets)
@@ -239,6 +239,10 @@ litellm_settings:
This sets a max budget of $10 USD for internal users when they sign up.
You can also manage these settings visually in the UI:
<Image img={require('../../img/default_user_settings_admin_ui.png')} style={{ width: '700px', height: 'auto' }} />
This budget only applies to personal keys created by that user - seen under `Default Team` on the UI.
<Image img={require('../../img/max_budget_for_internal_users.png')} style={{ width: '500px', height: 'auto' }} />
docs/my-website/docs/proxy/team_budgets.md
-185
View File
@@ -178,188 +178,3 @@ Expect to see this metric on prometheus to track the Remaining Budget for the te
```shell
litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06
```
### Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
1. Setup config.yaml
```yaml
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
#### ✨ [BETA] Set Priority / Reserve Quota
Reserve tpm/rpm capacity for projects in prod.
:::tip
Reserving tpm/rpm on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 100
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
priority_reservation: {"dev": 0, "prod": 1}
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
priority_reservation:
- Dict[str, float]
- str: can be any string
- float: from 0 to 1. Specify the % of tpm/rpm to reserve for keys of this priority.
**Start Proxy**
```
litellm --config /path/to/config.yaml
```
2. Create a key with that priority
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-D '{
"metadata": {"priority": "dev"} # 👈 KEY CHANGE
}'
```
**Expected Response**
```
{
...
"key": "sk-.."
}
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: sk-...' \ # 👈 key from step 2.
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
**Expected Response**
```
Key=... over available RPM=0. Model RPM=100, Active keys=None
```
docs/my-website/docs/proxy_api.md
+1 -1
View File
@@ -27,7 +27,7 @@ Email us @ krrish@berri.ai
## Supported Models for LiteLLM Key
These are the models that currently work with the "sk-litellm-.." keys.
For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/)
For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/) or check out [models.litellm.ai](https://models.litellm.ai/)
* OpenAI models - [OpenAI docs](./providers/openai.md)
* gpt-4
docs/my-website/docs/rerank.md
+2
View File
@@ -109,6 +109,8 @@ curl http://0.0.0.0:4000/rerank \
## **Supported Providers**
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Provider | Link to Usage |
|-------------|--------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
docs/my-website/docs/response_api.md
+40
View File
@@ -3,8 +3,11 @@ import TabItem from '@theme/TabItem';
# /responses [Beta]
LiteLLM provides a BETA endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
Requests to /chat/completions may be bridged here automatically when the provider lacks support for that endpoint. The model’s default `mode` determines how bridging works.(see `model_prices_and_context_window`)
| Feature | Supported | Notes |
|---------|-----------|--------|
| Cost Tracking | ✅ | Works with all supported models |
@@ -78,6 +81,43 @@ print(retrieved_response)
# retrieved_response = await litellm.aget_responses(response_id=response_id)
```
#### CANCEL a Response
You can cancel an in-progress response (if supported by the provider):
```python showLineNumbers title="Cancel Response by ID"
import litellm
# First, create a response
response = litellm.responses(
model="openai/o1-pro",
input="Tell me a three sentence bedtime story about a unicorn.",
max_output_tokens=100
)
# Get the response ID
response_id = response.id
# Cancel the response by ID
cancel_response = litellm.cancel_responses(
response_id=response_id
)
print(cancel_response)
# For async usage
# cancel_response = await litellm.acancel_responses(response_id=response_id)
```
**REST API:**
```bash
curl -X POST http://localhost:4000/v1/responses/response_id/cancel \
-H "Authorization: Bearer sk-1234"
```
This will attempt to cancel the in-progress response with the given ID.
**Note:** Not all providers support response cancellation. If unsupported, an error will be raised.
#### DELETE a Response
```python showLineNumbers title="Delete Response by ID"
import litellm
docs/my-website/docs/sdk/headers.md
+328
View File
@@ -0,0 +1,328 @@
# SDK Header Support
LiteLLM SDK provides comprehensive support for passing additional headers with API requests. This is essential for enterprise environments using API gateways, service meshes, and multi-tenant architectures.
## Overview
Headers can be passed to LiteLLM in three ways, with the following priority order:
1. **Request-specific headers** (highest priority)
2. **extra_headers parameter**
3. **Global litellm.headers** (lowest priority)
When the same header key is specified in multiple places, the higher priority value will be used.
## Usage Methods
### 1. Global Headers (litellm.headers)
Set headers that will be included in all API requests:
```python
import litellm
# Set global headers for all requests
litellm.headers = {
"X-API-Gateway-Key": "your-gateway-key",
"X-Company-ID": "acme-corp",
"X-Environment": "production"
}
# Now all completion calls will include these headers
response = litellm.completion(
model="claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": "Hello"}]
)
```
### 2. Per-Request Headers (extra_headers)
Pass headers for specific requests using the `extra_headers` parameter:
```python
import litellm
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_headers={
"X-Request-ID": "req-12345",
"X-Tenant-ID": "tenant-abc",
"X-Custom-Auth": "bearer-token-xyz"
}
)
```
### 3. Request Headers (headers parameter)
Use the `headers` parameter for the highest priority header control:
```python
import litellm
response = litellm.completion(
model="claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": "Hello"}],
headers={
"X-Priority-Header": "high-priority-value",
"Authorization": "Bearer custom-token"
}
)
```
### 4. Combining All Methods
You can combine all three methods. Headers will be merged with the priority order:
```python
import litellm
# Global headers (lowest priority)
litellm.headers = {
"X-Company-ID": "acme-corp",
"X-Shared-Header": "global-value"
}
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_headers={
"X-Request-ID": "req-12345",
"X-Shared-Header": "extra-value" # Overrides global
},
headers={
"X-Priority-Header": "important",
"X-Shared-Header": "request-value" # Overrides both global and extra
}
)
# Final headers sent to API:
# {
# "X-Company-ID": "acme-corp", # From global
# "X-Request-ID": "req-12345", # From extra_headers
# "X-Priority-Header": "important", # From headers
# "X-Shared-Header": "request-value" # From headers (highest priority)
# }
```
## Enterprise Use Cases
### API Gateway Integration (Apigee, Kong, AWS API Gateway)
```python
import litellm
# Set up headers for API gateway routing and authentication
litellm.headers = {
"X-API-Gateway-Key": "your-gateway-key",
"X-Route-Version": "v2"
}
# Per-tenant requests
response = litellm.completion(
model="claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": "Analyze this data"}],
extra_headers={
"X-Tenant-ID": "tenant-123",
"X-Department": "engineering"
}
)
```
### Service Mesh (Istio, Linkerd)
```python
import litellm
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_headers={
"X-Trace-ID": "trace-abc-123",
"X-Service-Name": "ai-service",
"X-Version": "1.2.3"
}
)
```
### Multi-Tenant SaaS Applications
```python
import litellm
def make_ai_request(user_id, tenant_id, content):
return litellm.completion(
model="claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": content}],
extra_headers={
"X-User-ID": user_id,
"X-Tenant-ID": tenant_id,
"X-Request-Time": str(int(time.time()))
}
)
# Usage
response = make_ai_request("user-456", "tenant-org-1", "Help me write code")
```
### Request Tracing and Debugging
```python
import litellm
import uuid
def traced_completion(model, messages, **kwargs):
trace_id = str(uuid.uuid4())
return litellm.completion(
model=model,
messages=messages,
extra_headers={
"X-Trace-ID": trace_id,
"X-Debug-Mode": "true",
"X-Source-Service": "my-app"
},
**kwargs
)
# Usage
response = traced_completion(
model="gpt-4",
messages=[{"role": "user", "content": "Debug this issue"}]
)
```
### Custom Authentication
```python
import litellm
def get_custom_auth_token():
# Your custom authentication logic
return "custom-auth-token"
response = litellm.completion(
model="claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": "Hello"}],
headers={
"X-Custom-Auth": get_custom_auth_token(),
"X-Auth-Type": "custom"
}
)
```
## Provider Support
Headers are supported across all LiteLLM providers including:
- **OpenAI** (GPT models)
- **Anthropic** (Claude models)
- **Cohere**
- **Hugging Face**
- **Custom providers**
- **Azure OpenAI**
- **AWS Bedrock**
- **Google Vertex AI**
Each provider will receive your custom headers along with their required authentication and API-specific headers.
## Best Practices
### 1. Use Meaningful Header Names
```python
# Good
extra_headers = {
"X-Request-ID": "req-12345",
"X-Tenant-ID": "org-456"
}
# Avoid
extra_headers = {
"custom1": "value1",
"h2": "value2"
}
```
### 2. Include Tracing Information
```python
extra_headers = {
"X-Trace-ID": trace_id,
"X-Span-ID": span_id,
"X-Service-Name": "ai-service"
}
```
### 3. Handle Sensitive Information Carefully
```python
# Don't log sensitive headers
import os
if os.getenv("ENVIRONMENT") != "production":
extra_headers["X-Debug-User"] = user_id
```
### 4. Use Environment-Specific Headers
```python
import os
environment = os.getenv("ENVIRONMENT", "development")
litellm.headers = {
"X-Environment": environment,
"X-Service-Version": os.getenv("SERVICE_VERSION", "unknown")
}
```
## Troubleshooting
### Headers Not Being Passed
If your headers aren't reaching the API:
1. **Check Header Names**: Ensure header names don't conflict with provider-specific headers
2. **Verify Priority**: Remember that `headers` > `extra_headers` > `litellm.headers`
3. **Test with Logging**: Enable verbose logging to see what headers are being sent
```python
import litellm
# Enable debug logging
litellm.set_verbose = True
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "test"}],
extra_headers={"X-Debug": "test"}
)
```
### Gateway or Proxy Issues
If using API gateways or proxies:
1. **Check Gateway Requirements**: Verify required headers for your gateway
2. **Test Direct vs Gateway**: Compare direct API calls vs gateway calls
3. **Validate Header Format**: Some gateways have header format requirements
## Security Considerations
1. **Don't Log Sensitive Headers**: Avoid logging authentication tokens or personal data
2. **Use HTTPS**: Always use secure connections when passing sensitive headers
3. **Validate Header Values**: Sanitize user-provided header values
4. **Rotate Keys**: Regularly rotate any API keys passed in headers
```python
import litellm
import re
def safe_header_value(value):
# Remove potentially dangerous characters
return re.sub(r'[^\w\-.]', '', str(value))
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_headers={
"X-User-ID": safe_header_value(user_id)
}
)
```
docs/my-website/img/default_user_settings_admin_ui.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 234 KiB

docs/my-website/release_notes/v1.76.0-stable/index.md
+1 -1
View File
@@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.76.0-stable - RPS Improvements"
title: "v1.76.0-stable - RPS Improvements"
slug: "v1-76-0"
date: 2025-08-23T10:00:00
authors:
docs/my-website/release_notes/v1.77.2-stable/index.md
+7 -7
View File
@@ -1,5 +1,5 @@
---
title: "[Pre-Release] v1.77.2-stable - Bedrock Batches API"
title: "v1.77.2-stable - Bedrock Batches API"
slug: "v1-77-2"
date: 2025-09-13T10:00:00
authors:
@@ -21,22 +21,22 @@ import TabItem from '@theme/TabItem';
## Deploy this version
:::info
This release is not yet live.
:::
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.2-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.2.post1
```
</TabItem>
docs/my-website/release_notes/v1.77.3-stable/index.md
+258
View File
@@ -0,0 +1,258 @@
---
title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.3.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.3
```
</TabItem>
</Tabs>
---
## Key Highlights
- **+550 RPS Performance Improvements** - Optimizations in request handling and object initialization.
- **Priority Quota Reservation** - Proxy admins can now reserve TPM/RPM capacity for specific keys.
## Priority Quota Reservation
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| SambaNova | `sambanova/deepseek-v3.1` | 128K | $0.90 | $0.90 | Chat completions |
| SambaNova | `sambanova/gpt-oss-120b` | 128K | $0.72 | $0.72 | Chat completions |
| OVHCloud | Various models | Varies | Contact provider | Contact provider | Chat completions |
| CompactifAI | Various models | Varies | Contact provider | Contact provider | Chat completions |
| TwelveLabs | `twelvelabs/marengo-embed-2.7` | 32K | $0.12 | $0.00 | Embeddings |
#### Features
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- New provider support with comprehensive model catalog - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider integration - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
- **[SambaNova](../../docs/providers/sambanova)**
- Added DeepSeek v3.1 and GPT-OSS-120B models - [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
- **[Bedrock](../../docs/providers/bedrock)**
- Cross-region inference profile cost calculation - [PR #14566](https://github.com/BerriAI/litellm/pull/14566)
- AWS external ID parameter support for authentication - [PR #14582](https://github.com/BerriAI/litellm/pull/14582)
- CountTokens API implementation - [PR #14557](https://github.com/BerriAI/litellm/pull/14557)
- Titan V2 encoding_format parameter support - [PR #14687](https://github.com/BerriAI/litellm/pull/14687)
- Nova Canvas image generation inference profiles - [PR #14578](https://github.com/BerriAI/litellm/pull/14578)
- Bedrock Batches API - batch processing support with file upload and request transformation - [PR #14618](https://github.com/BerriAI/litellm/pull/14618)
- Bedrock Twelve Labs embedding provider support - [PR #14697](https://github.com/BerriAI/litellm/pull/14697)
- **[Vertex AI](../../docs/providers/vertex)**
- Gemini labels field provider-aware filtering - [PR #14563](https://github.com/BerriAI/litellm/pull/14563)
- Gemini Batch API support - [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
- **[Volcengine](../../docs/providers/volcengine)**
- Fixed thinking parameters when disabled - [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
- **[Cohere](../../docs/providers/cohere)**
- Handle Generate API deprecation, default to chat endpoints - [PR #14676](https://github.com/BerriAI/litellm/pull/14676)
- **[TwelveLabs](../../docs/providers/twelvelabs)**
- Added Marengo Embed 2.7 embedding support - [PR #14674](https://github.com/BerriAI/litellm/pull/14674)
### Bug Fixes
- **[Bedrock](../../docs/providers/bedrock)**
- Empty arguments handling in tool call invocation - [PR #14583](https://github.com/BerriAI/litellm/pull/14583)
- **[Vertex AI](../../docs/providers/vertex)**
- Avoid deepcopy crash with non-pickleables in Gemini/Vertex - [PR #14418](https://github.com/BerriAI/litellm/pull/14418)
- **[XAI](../../docs/providers/xai)**
- Fix unsupported stop parameter for grok-code models - [PR #14565](https://github.com/BerriAI/litellm/pull/14565)
- **[Gemini](../../docs/providers/gemini)**
- Updated error message for Gemini API - [PR #14589](https://github.com/BerriAI/litellm/pull/14589)
- Fixed 2.5 Flash Image Preview model routing - [PR #14715](https://github.com/BerriAI/litellm/pull/14715)
- API key passing for token counting endpoints - [PR #14744](https://github.com/BerriAI/litellm/pull/14744)
#### New Provider Support
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- Complete provider integration with model catalog and authentication - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider support with documentation - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
---
## LLM API Endpoints
#### Features
- **[/responses](../../docs/response_api)**
- Added cancel endpoint support for non-admin users - [PR #14594](https://github.com/BerriAI/litellm/pull/14594)
- Improved response session handling and cold storage configuration with s3 - [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
- Added OpenAI & Azure /responses/cancel endpoint support - [PR #14561](https://github.com/BerriAI/litellm/pull/14561)
- **General**
- Enhanced rate limit error messages with details - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
- Middle-truncation for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Bugs
- **[/chat/completions](../../docs/completion/input)**
- Fixed completion chat ID handling - [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
- Prevent AttributeError for _get_tags_from_request_kwargs - [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
- **[/responses](../../docs/response_api)**
- Fixed cost calculation - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **General**
- Rate limiter AttributeError fix - [PR #14609](https://github.com/BerriAI/litellm/pull/14609)
---
## Spend Tracking, Budgets and Rate Limiting
- **Responses API Cost Calculation** fix - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **Anthropic Cache Token Pricing** - Separate 1-hour vs 5-minute cache creation costs - [PR #14620](https://github.com/BerriAI/litellm/pull/14620), [PR #14652](https://github.com/BerriAI/litellm/pull/14652)
- **Indochina Time Timezone** support for budget resets - [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
- **Soft Budget Alert Cache Issues** - Resolved soft budget alert cache issues - [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
- **Dynamic Rate Limiter v3** - Priority routing improvements - [PR #14734](https://github.com/BerriAI/litellm/pull/14734)
- **Enhanced Rate Limit Errors** - More detailed error messages - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
---
## Management Endpoints / UI
#### Features
- **Team Member Service Account Keys** - Allow team members to view keys they create - [PR #14619](https://github.com/BerriAI/litellm/pull/14619)
- **Default Budget for JWT Teams** - Auto-assign budgets to generated teams - [PR #14514](https://github.com/BerriAI/litellm/pull/14514)
- **SSO Access Control Groups** - Enhanced token info endpoint integration - [PR #14738](https://github.com/BerriAI/litellm/pull/14738)
- **Health Test Connect Protection** - Restrict access based on model creation permissions - [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Amazon Bedrock Guardrail Info View** - Enhanced logging visualization - [PR #14696](https://github.com/BerriAI/litellm/pull/14696)
#### Bug Fixes
- **SCIM v2** - Fix group PUSH and PUT operations for non-existent members - [PR #14581](https://github.com/BerriAI/litellm/pull/14581)
- **Guardrail View/Edit/Delete** behavior fixes - [PR #14622](https://github.com/BerriAI/litellm/pull/14622)
- **In-Memory Guardrail** update failures - [PR #14653](https://github.com/BerriAI/litellm/pull/14653)
---
## Logging / Guardrail Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Enhanced spend tracking metrics - [PR #14555](https://github.com/BerriAI/litellm/pull/14555)
- Stream support with is_streamed_request parameter - [PR #14673](https://github.com/BerriAI/litellm/pull/14673)
- Fixed tool calls metadata passing - [PR #14531](https://github.com/BerriAI/litellm/pull/14531)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Added logging support for Responses API - [PR #14597](https://github.com/BerriAI/litellm/pull/14597)
- **[Langsmith](../../docs/proxy/logging#langsmith)**
- Langsmith Sampling Rate - Key/Team-level tracing configuration - [PR #14740](https://github.com/BerriAI/litellm/pull/14740)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Multi-worker support improvements - [PR #14530](https://github.com/BerriAI/litellm/pull/14530)
- User email labels in monitoring - [PR #14520](https://github.com/BerriAI/litellm/pull/14520)
- **[Opik](../../docs/proxy/logging#opik)**
- Fixed timezone issue - [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
### Bug Fixes
- **[S3](../../docs/proxy/logging#s3-buckets)**
- Fixed 404 error when using s3_endpoint_url - [PR #14559](https://github.com/BerriAI/litellm/pull/14559)
#### Guardrails
- **Tool Permission Guardrail** - Fine-grained tool access control - [PR #14519](https://github.com/BerriAI/litellm/pull/14519)
- **Bedrock Guardrails** - Selective guarding support with runtime endpoint configuration - [PR #14575](https://github.com/BerriAI/litellm/pull/14575), [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Default Last Message** in guardrails - [PR #14640](https://github.com/BerriAI/litellm/pull/14640)
- **AWS exceptions handling despite 200 response** - [PR #14658](https://github.com/BerriAI/litellm/pull/14658)
#### New Integration
- **[PostHog](../../docs/observability/posthog)** - Complete observability integration for LiteLLM usage tracking and analytics - [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
---
## MCP Gateway
- **MCP Server Alias Parsing** - Multi-part URL path support - [PR #14558](https://github.com/BerriAI/litellm/pull/14558)
- **MCP Filter Recomputation** - After server deletion - [PR #14542](https://github.com/BerriAI/litellm/pull/14542)
- **MCP Gateway Tools List** improvements - [PR #14695](https://github.com/BerriAI/litellm/pull/14695)
---
## Performance / Loadbalancing / Reliability improvements
- **+500 RPS Performance Boost** when sending the `user` field - [PR #14616](https://github.com/BerriAI/litellm/pull/14616)
- **+50 RPS** by removing iscoroutine from hot path - [PR #14649](https://github.com/BerriAI/litellm/pull/14649)
- **7% reduction** in __init__ overhead - [PR #14689](https://github.com/BerriAI/litellm/pull/14689)
- **Generic Object Pool** implementation for better resource management - [PR #14702](https://github.com/BerriAI/litellm/pull/14702)
---
## General Proxy Improvements
- **Middle-Truncation** for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Security
- **Security Update** - Bump aiohttp==3.12.14, fix CVE-2025-53643 - [PR #14638](https://github.com/BerriAI/litellm/pull/14638)
---
## New Contributors
* @luisfucros made their first contribution in [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
* @hanakannzashi made their first contribution in [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
* @eliasto made their first contribution in [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
* @Rasmusafj made their first contribution in [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
* @LingXuanYin made their first contribution in [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
* @ronaldpereira made their first contribution in [PR #14613](https://github.com/BerriAI/litellm/pull/14613)
* @hula-la made their first contribution in [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
* @carlos-marchal-ph made their first contribution in [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
* @akraines made their first contribution in [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
* @mrFranklin made their first contribution in [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
* @tcx4c70 made their first contribution in [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
* @michaeltansg made their first contribution in [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
* @tosi29 made their first contribution in [PR #14725](https://github.com/BerriAI/litellm/pull/14725)
* @gmdfalk made their first contribution in [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
* @FelipeRodriguesGare made their first contribution in [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
* @mritunjaysharma394 made their first contribution in [PR #14678](https://github.com/BerriAI/litellm/pull/14678)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.2.rc.1...v1.77.3.rc.1)**
docs/my-website/sidebars.js
+193 -160
View File
@@ -57,31 +57,31 @@ const sidebars = {
type: "category",
label: "Alerting & Monitoring",
items: [
"proxy/prometheus",
"proxy/alerting",
"proxy/pagerduty"
].sort()
"proxy/pagerduty",
"proxy/prometheus"
]
},
{
type: "category",
label: "[Beta] Prompt Management",
items: [
"proxy/prompt_management",
"proxy/custom_prompt_management",
"proxy/native_litellm_prompt",
"proxy/custom_prompt_management"
].sort()
"proxy/prompt_management"
]
},
{
type: "category",
label: "AI Tools (OpenWebUI, Claude Code, etc.)",
items: [
"tutorials/openweb_ui",
"tutorials/openai_codex",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
"tutorials/github_copilot_integration",
"tutorials/claude_responses_api",
"tutorials/cost_tracking_coding",
"tutorials/github_copilot_integration",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
"tutorials/openai_codex",
"tutorials/openweb_ui"
]
},
@@ -111,41 +111,63 @@ const sidebars = {
label: "Setup & Deployment",
items: [
"proxy/quick_start",
"proxy/deploy",
"proxy/prod",
"proxy/cli",
"proxy/release_cycle",
"proxy/model_management",
"proxy/health",
"proxy/debugging",
"proxy/deploy",
"proxy/health",
"proxy/master_key_rotations",
"proxy/model_management",
"proxy/prod",
"proxy/release_cycle",
],
},
"proxy/demo",
{
type: "category",
label: "Admin UI",
items: [
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/custom_sso",
"proxy/model_hub",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui",
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_logs_sessions"
]
}
],
},
{
type: "category",
label: "Architecture",
items: ["proxy/architecture", "proxy/control_plane_and_data_plane", "proxy/db_info", "proxy/db_deadlocks", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch", "proxy/image_handling", "proxy/spend_logs_deletion"],
items: [
"proxy/architecture",
"proxy/control_plane_and_data_plane",
"proxy/db_deadlocks",
"proxy/db_info",
"proxy/image_handling",
"proxy/jwt_auth_arch",
"proxy/spend_logs_deletion",
"proxy/user_management_heirarchy",
"router_architecture"
],
},
{
type: "link",
label: "All Endpoints (Swagger)",
href: "https://litellm-api.up.railway.app/",
},
"proxy/enterprise",
"proxy/management_cli",
{
type: "category",
label: "Making LLM Requests",
items: [
"proxy/user_keys",
"proxy/clientside_auth",
"proxy/request_headers",
"proxy/response_headers",
"proxy/forward_client_headers",
"proxy/model_discovery",
],
},
"proxy/enterprise",
"proxy/management_cli",
{
type: "category",
label: "Authentication",
@@ -163,45 +185,25 @@ const sidebars = {
},
{
type: "category",
label: "Model Access",
label: "Budgets + Rate Limits",
items: [
"proxy/model_access",
"proxy/team_model_add"
]
},
{
type: "category",
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/model_hub",
"proxy/self_serve",
"proxy/public_teams",
"tutorials/scim_litellm",
"proxy/custom_sso",
"proxy/ui_credentials",
"proxy/ui/bulk_edit_users",
{
type: "category",
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_logs_sessions"
]
}
"proxy/customers",
"proxy/dynamic_rate_limit",
"proxy/rate_limit_tiers",
"proxy/team_budgets",
"proxy/temporary_budget_increase",
"proxy/users"
],
},
"proxy/caching",
{
type: "category",
label: "Spend Tracking",
items: ["proxy/cost_tracking", "proxy/custom_pricing", "proxy/billing",],
},
{
type: "category",
label: "Budgets + Rate Limits",
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/customers"],
label: "Create Custom Plugins",
description: "Modify requests, responses, and more",
items: [
"proxy/call_hooks",
"proxy/rules",
]
},
{
type: "link",
@@ -212,13 +214,32 @@ const sidebars = {
type: "category",
label: "Logging, Alerting, Metrics",
items: [
"proxy/dynamic_logging",
"proxy/logging",
"proxy/logging_spec",
"proxy/team_logging",
"proxy/dynamic_logging"
"proxy/team_logging"
],
},
{
type: "category",
label: "Making LLM Requests",
items: [
"proxy/user_keys",
"proxy/clientside_auth",
"proxy/request_headers",
"proxy/response_headers",
"proxy/forward_client_headers",
"proxy/model_discovery",
],
},
{
type: "category",
label: "Model Access",
items: [
"proxy/model_access",
"proxy/team_model_add"
]
},
{
type: "category",
label: "Secret Managers",
@@ -229,14 +250,13 @@ const sidebars = {
},
{
type: "category",
label: "Create Custom Plugins",
description: "Modify requests, responses, and more",
label: "Spend Tracking",
items: [
"proxy/call_hooks",
"proxy/rules",
]
"proxy/billing",
"proxy/cost_tracking",
"proxy/custom_pricing"
],
},
"proxy/caching",
]
},
{
@@ -250,6 +270,23 @@ const sidebars = {
slug: "/supported_endpoints",
},
items: [
"assistants",
{
type: "category",
label: "/audio",
items: [
"audio_transcription",
"text_to_speech",
]
},
{
type: "category",
label: "/batches",
items: [
"batches",
"proxy/managed_batches",
]
},
{
type: "category",
label: "/chat/completions",
@@ -266,57 +303,8 @@ const sidebars = {
"completion/http_handler_config",
],
},
"response_api",
"text_completion",
"embedding/supported_embedding",
"anthropic_unified",
"mcp",
"generateContent",
{
type: "category",
label: "/images",
items: [
"image_generation",
"image_edits",
"image_variations",
]
},
{
type: "category",
label: "/audio",
"items": [
"audio_transcription",
"text_to_speech",
]
},
{
type: "category",
label: "/vector_stores",
items: [
"vector_stores/search",
]
},
{
type: "category",
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
items: [
"pass_through/intro",
"pass_through/vertex_ai",
"pass_through/google_ai_studio",
"pass_through/cohere",
"pass_through/vllm",
"pass_through/mistral",
"pass_through/openai_passthrough",
"pass_through/anthropic_completion",
"pass_through/bedrock",
"pass_through/assembly_ai",
"pass_through/langfuse",
"proxy/pass_through",
],
},
"rerank",
"assistants",
{
type: "category",
label: "/files",
@@ -325,15 +313,6 @@ const sidebars = {
"proxy/litellm_managed_files",
],
},
{
type: "category",
label: "/batches",
items: [
"batches",
"proxy/managed_batches",
]
},
"realtime",
{
type: "category",
label: "/fine_tuning",
@@ -342,8 +321,48 @@ const sidebars = {
"proxy/managed_finetuning",
]
},
"generateContent",
"apply_guardrail",
{
type: "category",
label: "/images",
items: [
"image_edits",
"image_generation",
"image_variations",
]
},
"mcp",
"moderation",
"apply_guardrail",
{
type: "category",
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
items: [
"pass_through/intro",
"pass_through/anthropic_completion",
"pass_through/assembly_ai",
"pass_through/bedrock",
"pass_through/cohere",
"pass_through/google_ai_studio",
"pass_through/langfuse",
"pass_through/mistral",
"pass_through/openai_passthrough",
"pass_through/vertex_ai",
"pass_through/vllm",
"proxy/pass_through"
]
},
"realtime",
"rerank",
"response_api",
"anthropic_unified",
{
type: "category",
label: "/vector_stores",
items: [
"vector_stores/search",
]
},
],
},
{
@@ -383,6 +402,7 @@ const sidebars = {
items: [
"providers/azure_ai",
"providers/azure_ai_img",
"providers/azure_ai_img_edit",
]
},
{
@@ -392,6 +412,7 @@ const sidebars = {
"providers/vertex",
"providers/vertex_partner",
"providers/vertex_image",
"providers/vertex_batch",
]
},
{
@@ -498,32 +519,32 @@ const sidebars = {
type: "category",
label: "Guides",
items: [
"exception_mapping",
"completion/audio",
"completion/batching",
"completion/computer_use",
"completion/document_understanding",
"completion/drop_params",
"completion/function_call",
"completion/image_generation_chat",
"completion/json_mode",
"completion/knowledgebase",
"completion/message_trimming",
"completion/model_alias",
"completion/mock_requests",
"completion/predict_outputs",
"completion/prefix",
"completion/prompt_caching",
"completion/prompt_formatting",
"completion/reliable_completions",
"completion/stream",
"completion/provider_specific_params",
"completion/vision",
"completion/web_search",
"exception_mapping",
"guides/finetuned_models",
"guides/security_settings",
"completion/audio",
"completion/image_generation_chat",
"completion/web_search",
"completion/document_understanding",
"completion/vision",
"completion/json_mode",
"reasoning_content",
"completion/computer_use",
"completion/prompt_caching",
"completion/predict_outputs",
"completion/knowledgebase",
"completion/prefix",
"completion/drop_params",
"completion/prompt_formatting",
"completion/stream",
"completion/message_trimming",
"completion/function_call",
"completion/model_alias",
"completion/batching",
"completion/mock_requests",
"completion/reliable_completions",
"proxy/veo_video_generation",
"reasoning_content"
]
},
@@ -536,25 +557,37 @@ const sidebars = {
description: "Learn how to load balance, route, and set fallbacks for your LLM requests",
slug: "/routing-load-balancing",
},
items: ["routing", "scheduler", "proxy/load_balancing", "proxy/reliability", "proxy/timeout", "proxy/auto_routing", "proxy/tag_routing", "proxy/provider_budget_routing", "wildcard_routing"],
items: [
"routing",
"scheduler",
"proxy/auto_routing",
"proxy/load_balancing",
"proxy/provider_budget_routing",
"proxy/reliability",
"proxy/tag_routing",
"proxy/timeout",
"wildcard_routing"
],
},
{
type: "category",
label: "LiteLLM Python SDK",
items: [
"set_keys",
"budget_manager",
"caching/all_caches",
"completion/token_usage",
"sdk/headers",
"sdk_custom_pricing",
"embedding/async_embedding",
"embedding/moderation",
"budget_manager",
"caching/all_caches",
"migration",
"sdk_custom_pricing",
{
type: "category",
label: "LangChain, LlamaIndex, Instructor Integration",
items: ["langchain/langchain", "tutorials/instructor"],
},
}
],
},
enterprise/litellm_enterprise/integrations/prometheus.py
-1
View File
@@ -2328,7 +2328,6 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
"tag_Service_web_app_v1": "false",
}
"""
import re
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py
-1
View File
@@ -2,7 +2,6 @@
Enterprise internal user management endpoints
"""
import os
from fastapi import APIRouter, Depends, HTTPException
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
+1 -1
View File
@@ -11,7 +11,7 @@ All /vector_store management endpoints
import copy
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi import APIRouter, Depends, HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
examples/sdk_headers_example.py
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env python3
"""
Example demonstrating LiteLLM SDK header support for enterprise environments.
This example shows how to use additional headers with API gateways, service meshes,
and multi-tenant architectures.
"""
import litellm
import os
from typing import Dict, Any
def example_global_headers():
"""Example: Set global headers for all requests"""
print("=== Global Headers Example ===")
# Set global headers that will be included in all API requests
litellm.headers = {
"X-API-Gateway-Key": "your-gateway-key-here",
"X-Company-ID": "acme-corp",
"X-Environment": "production"
}
print("Global headers set:", litellm.headers)
# These headers will now be included in all completion calls
# (Note: This example doesn't actually make API calls)
print("Global headers will be included in all subsequent completion() calls")
def example_per_request_headers():
"""Example: Using extra_headers for specific requests"""
print("\n=== Per-Request Headers Example ===")
headers_to_send = {
"X-Request-ID": "req-12345",
"X-Tenant-ID": "tenant-abc",
"X-Custom-Auth": "bearer-token-xyz"
}
print("Per-request headers:", headers_to_send)
# Example of how you would use extra_headers in a real call
# response = litellm.completion(
# model="claude-3-5-sonnet-latest",
# messages=[{"role": "user", "content": "Hello"}],
# extra_headers=headers_to_send
# )
def example_header_priority():
"""Example: Demonstrating header priority and merging"""
print("\n=== Header Priority Example ===")
# Set global headers
litellm.headers = {
"X-Company-ID": "acme-corp",
"X-Shared-Header": "global-value"
}
# Headers that would be sent in a request
extra_headers = {
"X-Request-ID": "req-12345",
"X-Shared-Header": "extra-value" # Overrides global
}
request_headers = {
"X-Priority-Header": "important",
"X-Shared-Header": "request-value" # Overrides both global and extra
}
print("Global headers:", litellm.headers)
print("Extra headers:", extra_headers)
print("Request headers:", request_headers)
print("\nFinal headers would be:")
print(" X-Company-ID: acme-corp (from global)")
print(" X-Request-ID: req-12345 (from extra)")
print(" X-Priority-Header: important (from request)")
print(" X-Shared-Header: request-value (request wins - highest priority)")
def example_enterprise_api_gateway():
"""Example: Enterprise API Gateway scenario"""
print("\n=== Enterprise API Gateway Example ===")
# Simulate enterprise environment with Apigee or similar
gateway_config = {
"X-API-Gateway-Key": os.getenv("API_GATEWAY_KEY", "demo-key"),
"X-Route-Version": "v2",
"X-Rate-Limit-Group": "premium"
}
# Set gateway headers globally
litellm.headers = gateway_config
print("Gateway headers configured:", gateway_config)
# Function to make tenant-specific requests
def make_tenant_request(tenant_id: str, user_id: str, content: str) -> Dict[str, Any]:
"""Make an AI request with tenant-specific headers"""
tenant_headers = {
"X-Tenant-ID": tenant_id,
"X-User-ID": user_id,
"X-Request-Time": "2024-01-01T00:00:00Z",
"X-Service-Name": "ai-assistant"
}
print(f"Making request for tenant {tenant_id}, user {user_id}")
print("Tenant-specific headers:", tenant_headers)
# In a real scenario, this would make the actual API call:
# return litellm.completion(
# model="claude-3-5-sonnet-latest",
# messages=[{"role": "user", "content": content}],
# extra_headers=tenant_headers
# )
# For demo purposes, return mock data
return {"mock": "response", "headers_used": {**gateway_config, **tenant_headers}}
# Example usage
result = make_tenant_request("tenant-123", "user-456", "Analyze this data")
print("Response:", result)
def example_service_mesh():
"""Example: Service mesh integration (Istio, Linkerd)"""
print("\n=== Service Mesh Example ===")
service_mesh_headers = {
"X-Trace-ID": "trace-abc-123",
"X-Span-ID": "span-def-456",
"X-Service-Name": "ai-service",
"X-Version": "1.2.3",
"X-Cluster": "prod-us-west-2"
}
print("Service mesh headers:", service_mesh_headers)
# Example of using these headers for distributed tracing
# response = litellm.completion(
# model="gpt-4",
# messages=[{"role": "user", "content": "Hello"}],
# extra_headers=service_mesh_headers
# )
def example_debugging_and_monitoring():
"""Example: Request debugging and monitoring"""
print("\n=== Debugging and Monitoring Example ===")
import uuid
import time
# Generate unique identifiers for request tracking
trace_id = str(uuid.uuid4())
request_id = f"req-{int(time.time())}"
debug_headers = {
"X-Trace-ID": trace_id,
"X-Request-ID": request_id,
"X-Debug-Mode": "true",
"X-Source-Service": "customer-support-bot",
"X-Request-Priority": "high"
}
print("Debug headers:", debug_headers)
print(f"Trace ID: {trace_id}")
print(f"Request ID: {request_id}")
# These headers help with:
# 1. Distributed tracing across services
# 2. Request correlation in logs
# 3. Debug mode enablement
# 4. Priority-based routing
if __name__ == "__main__":
print("LiteLLM SDK Header Support Examples")
print("=" * 50)
example_global_headers()
example_per_request_headers()
example_header_priority()
example_enterprise_api_gateway()
example_service_mesh()
example_debugging_and_monitoring()
print("\n" + "=" * 50)
print("All examples completed!")
print("\nTo use in your application:")
print("1. Set litellm.headers for global headers")
print("2. Use extra_headers parameter for request-specific headers")
print("3. Use headers parameter for highest priority headers")
print("4. Headers are merged with priority: headers > extra_headers > litellm.headers")
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20250918083359_drop_spec_version_column_from_mcp_table/migration.sql
+8
View File
@@ -0,0 +1,8 @@
/*
Warnings:
- You are about to drop the column `spec_version` on the `LiteLLM_MCPServerTable` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "public"."LiteLLM_MCPServerTable" DROP COLUMN "spec_version";
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
-1
View File
@@ -171,7 +171,6 @@ model LiteLLM_MCPServerTable {
description String?
url String?
transport String @default("sse")
spec_version String @default("2025-03-26")
auth_type String?
created_at DateTime? @default(now()) @map("created_at")
created_by String?
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.18"
version = "0.2.19"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.18"
version = "0.2.19"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+12
View File
@@ -60,6 +60,7 @@ from litellm.constants import (
empower_models,
together_ai_models,
baseten_models,
WANDB_MODELS,
REPEATED_STREAMING_CHUNK_LIMIT,
request_timeout,
open_ai_embedding_models,
@@ -117,6 +118,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"logfire",
"literalai",
"dynamic_rate_limiter",
"dynamic_rate_limiter_v3",
"langsmith",
"prometheus",
"otel",
@@ -241,6 +243,7 @@ novita_api_key: Optional[str] = None
snowflake_key: Optional[str] = None
gradient_ai_api_key: Optional[str] = None
nebius_key: Optional[str] = None
wandb_key: Optional[str] = None
heroku_key: Optional[str] = None
cometapi_key: Optional[str] = None
ovhcloud_key: Optional[str] = None
@@ -523,6 +526,7 @@ cometapi_models: Set = set()
oci_models: Set = set()
vercel_ai_gateway_models: Set = set()
volcengine_models: Set = set()
wandb_models: Set = set(WANDB_MODELS)
ovhcloud_models: Set = set()
ovhcloud_embedding_models: Set = set()
@@ -739,6 +743,8 @@ def add_known_models():
oci_models.add(key)
elif value.get("litellm_provider") == "volcengine":
volcengine_models.add(key)
elif value.get("litellm_provider") == "wandb":
wandb_models.add(key)
elif value.get("litellm_provider") == "ovhcloud":
ovhcloud_models.add(key)
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
@@ -837,6 +843,7 @@ model_list = list(
| heroku_models
| vercel_ai_gateway_models
| volcengine_models
| wandb_models
| ovhcloud_models
)
@@ -919,6 +926,7 @@ models_by_provider: dict = {
"cometapi": cometapi_models,
"oci": oci_models,
"volcengine": volcengine_models,
"wandb": wandb_models,
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
}
@@ -1258,6 +1266,7 @@ from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig
from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
@@ -1334,5 +1343,8 @@ disable_hf_tokenizer_download: Optional[bool] = (
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route
litellm/constants.py
+37
View File
@@ -313,6 +313,7 @@ LITELLM_CHAT_PROVIDERS = [
"morph",
"lambda_ai",
"vercel_ai_gateway",
"wandb",
"ovhcloud",
]
@@ -448,6 +449,7 @@ openai_compatible_endpoints: List = [
"https://api.lambda.ai/v1",
"https://api.hyperbolic.xyz/v1",
"https://ai-gateway.vercel.sh/v1",
"https://api.inference.wandb.ai/v1",
]
@@ -492,6 +494,7 @@ openai_compatible_providers: List = [
"hyperbolic",
"vercel_ai_gateway",
"aiml",
"wandb",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@@ -507,6 +510,7 @@ openai_text_completion_compatible_providers: List = (
"v0",
"lambda_ai",
"hyperbolic",
"wandb",
]
)
_openai_like_providers: List = [
@@ -757,6 +761,38 @@ nebius_embedding_models: set = set(
]
)
WANDB_MODELS: set = set(
[
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# zai-org models
"zai-org/GLM-4.5",
# Qwen models
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Thinking-2507",
# moonshotai
"moonshotai/Kimi-K2-Instruct",
# meta models
"meta-llama/Llama-3.1-8B-Instruct",
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
# deepseek-ai
"deepseek-ai/DeepSeek-V3.1",
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
# microsoft
"microsoft/Phi-4-mini-instruct",
]
)
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
"anthropic",
@@ -947,6 +983,7 @@ HEALTH_CHECK_TIMEOUT_SECONDS = int(
os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)
) # 60 seconds
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
litellm/cost_calculator.py
+17 -1
View File
@@ -148,6 +148,8 @@ def cost_per_token( # noqa: PLR0915
### CALL TYPE ###
call_type: CallTypesLiteral = "completion",
audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> Tuple[float, float]: # type: ignore
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -278,6 +280,7 @@ def cost_per_token( # noqa: PLR0915
model=model_without_prefix,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
)
return prompt_cost, completion_cost
@@ -327,7 +330,9 @@ def cost_per_token( # noqa: PLR0915
elif custom_llm_provider == "bedrock":
return bedrock_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "openai":
return openai_cost_per_token(model=model, usage=usage_block)
return openai_cost_per_token(
model=model, usage=usage_block, service_tier=service_tier
)
elif custom_llm_provider == "databricks":
return databricks_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "fireworks_ai":
@@ -348,6 +353,7 @@ def cost_per_token( # noqa: PLR0915
from litellm.llms.dashscope.cost_calculator import (
cost_per_token as dashscope_cost_per_token,
)
return dashscope_cost_per_token(model=model, usage=usage_block)
else:
model_info = _cached_get_model_info_helper(
@@ -606,6 +612,8 @@ def completion_cost( # noqa: PLR0915
litellm_model_name: Optional[str] = None,
router_model_id: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
@@ -659,6 +667,10 @@ def completion_cost( # noqa: PLR0915
)
rerank_billed_units: Optional[RerankBilledUnits] = None
# Extract service_tier from optional_params if not provided directly
if service_tier is None and optional_params is not None:
service_tier = optional_params.get("service_tier")
selected_model = _select_model_name_for_cost_calc(
model=model,
completion_response=completion_response,
@@ -909,6 +921,7 @@ def completion_cost( # noqa: PLR0915
call_type=cast(CallTypesLiteral, call_type),
audio_transcription_file_duration=audio_transcription_file_duration,
rerank_billed_units=rerank_billed_units,
service_tier=service_tier,
)
_final_cost = (
prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
@@ -1003,6 +1016,8 @@ def response_cost_calculator(
litellm_model_name: Optional[str] = None,
router_model_id: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> float:
"""
Returns
@@ -1036,6 +1051,7 @@ def response_cost_calculator(
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
)
return response_cost
except Exception as e:
litellm/experimental_mcp_client/client.py
+28 -31
View File
@@ -19,8 +19,6 @@ from litellm._logging import verbose_logger
from litellm.types.mcp import (
MCPAuth,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPStdioConfig,
MCPTransport,
MCPTransportType,
@@ -48,7 +46,6 @@ class MCPClient:
auth_value: Optional[str] = None,
timeout: float = 60.0,
stdio_config: Optional[MCPStdioConfig] = None,
protocol_version: MCPSpecVersionType = MCPSpecVersion.jun_2025,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@@ -62,7 +59,6 @@ class MCPClient:
self._session_ctx = None
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.protocol_version: MCPSpecVersionType = protocol_version
# handle the basic auth value if provided
if auth_value:
@@ -84,22 +80,24 @@ class MCPClient:
"""Initialize the transport and session."""
if self._session:
return # Already connected
try:
if self.transport_type == MCPTransport.stdio:
# For stdio transport, use stdio_client with command-line parameters
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {})
env=self.stdio_config.get("env", {}),
)
self._transport_ctx = stdio_client(server_params)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
elif self.transport_type == MCPTransport.sse:
@@ -110,7 +108,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
else: # http
@@ -121,7 +121,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
except ValueError as e:
@@ -184,8 +186,10 @@ class MCPClient:
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers = {}
headers = {
"MCP-Protocol-Version": "2025-06-18"
}
if self._mcp_auth_value:
if self.auth_type == MCPAuth.bearer_token:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
@@ -196,18 +200,8 @@ class MCPClient:
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
# Handle protocol version - it might be a string or enum
if hasattr(self.protocol_version, 'value'):
# It's an enum
protocol_version_str = self.protocol_version.value
else:
# It's a string
protocol_version_str = str(self.protocol_version)
headers["MCP-Protocol-Version"] = protocol_version_str
return headers
async def list_tools(self) -> List[MCPTool]:
"""List available tools from the server."""
if not self._session:
@@ -216,7 +210,7 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return []
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return []
@@ -245,17 +239,20 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")],
isError=True
content=[TextContent(type="text", text=f"{str(e)}")], isError=True
)
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return MCPCallToolResult(
content=[TextContent(type="text", text="MCP client session is not initialized")],
content=[
TextContent(
type="text", text="MCP client session is not initialized"
)
],
isError=True,
)
try:
tool_result = await self._session.call_tool(
name=call_tool_request_params.name,
@@ -270,8 +267,8 @@ class MCPClient:
await self.disconnect()
# Return a default error result instead of raising
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")], # Empty content for error case
content=[
TextContent(type="text", text=f"{str(e)}")
], # Empty content for error case
isError=True,
)
litellm/files/main.py
+27 -1
View File
@@ -731,7 +731,7 @@ def file_list(
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@@ -887,6 +887,32 @@ def file_content(
client=client,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base = optional_params.api_base or ""
vertex_ai_project = (
optional_params.vertex_project
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.vertex_location
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
response = vertex_ai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
timeout=timeout,
max_retries=optional_params.max_retries,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai'.".format(
litellm/integrations/langsmith.py
+34 -24
View File
@@ -39,6 +39,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_api_key: Optional[str] = None,
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_sampling_rate: Optional[float] = None,
**kwargs,
):
self.flush_lock = asyncio.Lock()
@@ -49,7 +50,8 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url=langsmith_base_url,
)
self.sampling_rate: float = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
langsmith_sampling_rate
or float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
@@ -76,26 +78,14 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url: Optional[str] = None,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
if _credentials_api_key is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_api_key=None."
)
_credentials_project = (
langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
)
if _credentials_project is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_project=None."
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
if _credentials_base_url is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_base_url=None."
)
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
@@ -200,12 +190,7 @@ class LangsmithLogger(CustomBatchLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
)
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -219,6 +204,7 @@ class LangsmithLogger(CustomBatchLogger):
kwargs,
response_obj,
)
credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
data = self._prepare_log_data(
kwargs=kwargs,
@@ -245,7 +231,7 @@ class LangsmithLogger(CustomBatchLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -286,7 +272,7 @@ class LangsmithLogger(CustomBatchLogger):
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -417,6 +403,17 @@ class LangsmithLogger(CustomBatchLogger):
for queue_object in self.log_queue:
credentials = queue_object["credentials"]
# if credential missing, skip - log warning
if (
credentials["LANGSMITH_API_KEY"] is None
or credentials["LANGSMITH_PROJECT"] is None
):
verbose_logger.warning(
"Langsmith Logging - credentials missing - api_key: %s, project: %s",
credentials["LANGSMITH_API_KEY"],
credentials["LANGSMITH_PROJECT"],
)
continue
key = CredentialsKey(
api_key=credentials["LANGSMITH_API_KEY"],
project=credentials["LANGSMITH_PROJECT"],
@@ -432,6 +429,19 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
sampling_rate: float = self.sampling_rate
if standard_callback_dynamic_params is not None:
_sampling_rate = standard_callback_dynamic_params.get(
"langsmith_sampling_rate"
)
if _sampling_rate is not None:
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(
self, kwargs: Dict[str, Any]
) -> LangsmithCredentialsObject:
@@ -442,9 +452,9 @@ class LangsmithLogger(CustomBatchLogger):
Otherwise, use the default credentials.
"""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
credentials = self.get_credentials_from_env(
langsmith_api_key=standard_callback_dynamic_params.get(
litellm/integrations/opik/opik.py
+5 -4
View File
@@ -3,6 +3,7 @@ Opik Logger that logs LLM events to an Opik server
"""
import asyncio
from datetime import timezone
import json
import traceback
from typing import Dict, List
@@ -291,8 +292,8 @@ class OpikLogger(CustomBatchLogger):
"project_name": project_name,
"id": trace_id,
"name": trace_name,
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,
@@ -312,8 +313,8 @@ class OpikLogger(CustomBatchLogger):
"parent_span_id": parent_span_id,
"name": span_name,
"type": "llm",
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,
litellm/litellm_core_utils/cli_token_utils.py
+58
View File
@@ -0,0 +1,58 @@
"""
CLI Token Utilities
SDK-level utilities for reading CLI authentication tokens.
This module has no dependencies on proxy code and can be safely imported at the SDK level.
"""
import json
import os
from pathlib import Path
from typing import Optional
def get_cli_token_file_path() -> str:
"""Get the path to the CLI token file"""
home_dir = Path.home()
config_dir = home_dir / ".litellm"
return str(config_dir / "token.json")
def load_cli_token() -> Optional[dict]:
"""Load CLI token data from file"""
token_file = get_cli_token_file_path()
if not os.path.exists(token_file):
return None
try:
with open(token_file, 'r') as f:
return json.load(f)
except (json.JSONDecodeError, IOError):
return None
def get_litellm_gateway_api_key() -> Optional[str]:
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
and returns the API key for use in Python scripts.
Returns:
str: The API key if found, None otherwise
Example:
>>> import litellm
>>> api_key = litellm.get_litellm_gateway_api_key()
>>> if api_key:
>>> response = litellm.completion(
>>> model="gpt-3.5-turbo",
>>> messages=[{"role": "user", "content": "Hello"}],
>>> api_key=api_key,
>>> base_url="https://your-proxy.com/v1"
>>> )
"""
token_data = load_cli_token()
if token_data and 'key' in token_data:
return token_data['key']
return None
litellm/litellm_core_utils/custom_logger_registry.py
+2
View File
@@ -47,6 +47,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
VectorStorePreCallHook,
)
from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
class CustomLoggerRegistry:
@@ -86,6 +87,7 @@ class CustomLoggerRegistry:
"s3_v2": S3Logger,
"aws_sqs": SQSLogger,
"dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
"dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
"vector_store_pre_call_hook": VectorStorePreCallHook,
"dotprompt": DotpromptManager,
"cloudzero": CloudZeroLogger,
litellm/litellm_core_utils/duration_parser.py
+1
View File
@@ -158,6 +158,7 @@ def _setup_timezone(
"US/Eastern": timezone(timedelta(hours=-4)), # EDT
"US/Pacific": timezone(timedelta(hours=-7)), # PDT
"Asia/Kolkata": timezone(timedelta(hours=5, minutes=30)), # IST
"Asia/Bangkok": timezone(timedelta(hours=7)), # ICT (Indochina Time)
"Europe/London": timezone(timedelta(hours=1)), # BST
"UTC": timezone.utc,
}
litellm/litellm_core_utils/exception_mapping_utils.py
+62 -98
View File
@@ -6,6 +6,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import LlmProviders
from ..exceptions import (
APIConnectionError,
@@ -762,7 +763,7 @@ def exception_type( # type: ignore # noqa: PLR0915
error_str += "XXXXXXX" + '"'
raise AuthenticationError(
message=f"{custom_llm_provider}Exception: Authentication Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@@ -771,14 +772,14 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "model's maximum context limit" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"{custom_llm_provider}Exception: Context Window Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
elif "token_quota_reached" in error_str:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception: Rate Limit Errror - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@@ -789,14 +790,14 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif "model_no_support_for_function" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -804,7 +805,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -814,28 +815,28 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -846,7 +847,7 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -854,7 +855,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -862,7 +863,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 503:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -870,7 +871,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 504: # gateway timeout error
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -1168,9 +1169,9 @@ def exception_type( # type: ignore # noqa: PLR0915
exception_status_code=original_exception.status_code,
)
elif (
custom_llm_provider == "vertex_ai"
or custom_llm_provider == "vertex_ai_beta"
or custom_llm_provider == "gemini"
custom_llm_provider == LlmProviders.VERTEX_AI
or custom_llm_provider == LlmProviders.VERTEX_AI_BETA
or custom_llm_provider == LlmProviders.GEMINI
):
if (
"Vertex AI API has not been used in project" in error_str
@@ -1178,9 +1179,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"litellm.BadRequestError: VertexAIException - {error_str}",
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
request=httpx.Request(
@@ -1193,7 +1194,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"VertexException - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
@@ -1203,9 +1204,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=500,
content=str(original_exception),
@@ -1216,7 +1217,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "API key not valid." in error_str:
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -1224,9 +1225,9 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "403" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=403,
request=httpx.Request(
@@ -1243,9 +1244,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise ContentPolicyViolationError(
message=f"VertexAIException ContentPolicyViolationError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@@ -1264,9 +1265,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@@ -1282,18 +1283,18 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@@ -1306,21 +1307,35 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 401:
exception_mapping_worked = True
raise AuthenticationError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 403:
exception_mapping_worked = True
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=httpx.Response(
status_code=403,
request=httpx.Request(
method="POST",
url="https://cloud.google.com/vertex-ai/",
),
),
)
if original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -1328,9 +1343,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@@ -1343,9 +1358,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"VertexAIException InternalServerError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=500,
@@ -1353,71 +1368,20 @@ def exception_type( # type: ignore # noqa: PLR0915
request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
if original_exception.status_code == 503:
if original_exception.status_code == 502:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"VertexAIException - {original_exception.message}",
raise APIConnectionError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
if "503 Getting metadata" in error_str:
# auth errors look like this
# 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate.
exception_mapping_worked = True
raise BadRequestError(
message="GeminiException - Invalid api key",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"504 Deadline expired before operation could complete." in error_str
or "504 Deadline Exceeded" in error_str
):
exception_mapping_worked = True
raise Timeout(
message=f"GeminiException - {original_exception.message}",
model=model,
llm_provider="palm",
exception_status_code=original_exception.status_code,
)
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"GeminiException - {error_str}",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"500 An internal error has occurred." in error_str
or "list index out of range" in error_str
):
exception_mapping_worked = True
raise APIError(
status_code=getattr(original_exception, "status_code", 500),
message=f"GeminiException - {original_exception.message}",
llm_provider="palm",
model=model,
request=httpx.Response(
status_code=429,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
),
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
if original_exception.status_code == 503:
exception_mapping_worked = True
raise BadRequestError(
message=f"GeminiException - {error_str}",
raise ServiceUnavailableError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
# Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes
elif custom_llm_provider == "cloudflare":
if "Authentication error" in error_str:
exception_mapping_worked = True
litellm/litellm_core_utils/get_llm_provider_logic.py
+10
View File
@@ -252,6 +252,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://ai-gateway.vercel.sh/v1":
custom_llm_provider = "vercel_ai_gateway"
dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
elif endpoint == "https://api.inference.wandb.ai/v1":
custom_llm_provider = "wandb"
dynamic_api_key = get_secret_str("WANDB_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(
@@ -773,6 +776,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "wandb":
api_base = (
api_base
or get_secret("WANDB_API_BASE")
or "https://api.inference.wandb.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))
litellm/litellm_core_utils/get_supported_openai_params.py
+3
View File
@@ -149,6 +149,9 @@ def get_supported_openai_params( # noqa: PLR0915
elif custom_llm_provider == "nebius":
if request_type == "chat_completion":
return litellm.NebiusConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "wandb":
if request_type == "chat_completion":
return litellm.WandbConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "replicate":
return litellm.ReplicateConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "huggingface":
litellm/litellm_core_utils/litellm_logging.py
+45 -4
View File
@@ -1228,6 +1228,9 @@ class Logging(LiteLLMLoggingBaseClass):
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
"litellm_logging_obj": self,
"service_tier": self.optional_params.get("service_tier")
if self.optional_params
else None,
}
except Exception as e: # error creating kwargs for cost calculation
debug_info = StandardLoggingModelCostFailureDebugInformation(
@@ -3444,6 +3447,30 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj)
return dynamic_rate_limiter_obj # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
if internal_usage_cache is None:
raise Exception(
"Internal Error: Cache cannot be empty - internal_usage_cache={}".format(
internal_usage_cache
)
)
dynamic_rate_limiter_obj_v3 = _PROXY_DynamicRateLimitHandlerV3(
internal_usage_cache=internal_usage_cache
)
if llm_router is not None and isinstance(llm_router, litellm.Router):
dynamic_rate_limiter_obj_v3.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj_v3)
return dynamic_rate_limiter_obj_v3 # type: ignore
elif logging_integration == "langtrace":
if "LANGTRACE_API_KEY" not in os.environ:
raise ValueError("LANGTRACE_API_KEY not found in environment variables")
@@ -3707,6 +3734,14 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandler):
return callback # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
elif logging_integration == "langtrace":
from litellm.integrations.opentelemetry import OpenTelemetry
@@ -4158,16 +4193,22 @@ class StandardLoggingPayloadSetup:
# Get the actual s3_path from the configured cold storage logger instance
s3_path = "" # default value
# Try to get the actual logger instance from the logger name
try:
custom_logger = litellm.logging_callback_manager.get_active_custom_logger_for_callback_name(configured_cold_storage_logger)
if custom_logger and hasattr(custom_logger, 's3_path') and custom_logger.s3_path:
custom_logger = litellm.logging_callback_manager.get_active_custom_logger_for_callback_name(
configured_cold_storage_logger
)
if (
custom_logger
and hasattr(custom_logger, "s3_path")
and custom_logger.s3_path
):
s3_path = custom_logger.s3_path
except Exception:
# If any error occurs in getting the logger instance, use default empty s3_path
pass
s3_object_key = get_s3_object_key(
s3_path=s3_path, # Use actual s3_path from logger configuration
team_alias_prefix="", # Don't split by team alias for cold storage
litellm/litellm_core_utils/llm_cost_calc/utils.py
+301 -115
View File
@@ -1,15 +1,17 @@
# What is this?
## Helper utilities for cost_per_token()
from typing import Any, Literal, Optional, Tuple, cast
from typing import Any, Literal, Optional, Tuple, TypedDict, cast
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
ImageResponse,
ModelInfo,
PassthroughCallTypes,
ServiceTier,
Usage,
)
from litellm.utils import get_model_info
@@ -113,9 +115,31 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str:
"""
Get the appropriate cost key based on service tier.
Args:
base_key: The base cost key (e.g., "input_cost_per_token")
service_tier: The service tier ("flex", "priority", or None for standard)
Returns:
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
"""
if service_tier is None:
return base_key
# Only use service tier specific keys for "flex" and "priority"
if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]:
return f"{base_key}_{service_tier.lower()}"
# For any other service tier, use standard pricing
return base_key
def _get_token_base_cost(
model_info: ModelInfo, usage: Usage
) -> Tuple[float, float, float, float]:
model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
) -> Tuple[float, float, float, float, float]:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@@ -125,18 +149,26 @@ def _get_token_base_cost(
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, "input_cost_per_token")
# Get service tier aware cost keys
input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier)
output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier)
cache_creation_cost_key = _get_service_tier_cost_key(
"cache_creation_input_token_cost", service_tier
)
completion_base_cost = cast(
float, _get_cost_per_unit(model_info, "output_cost_per_token")
cache_read_cost_key = _get_service_tier_cost_key(
"cache_read_input_token_cost", service_tier
)
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key))
cache_creation_cost = cast(
float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")
float, _get_cost_per_unit(model_info, cache_creation_cost_key)
)
cache_read_cost = cast(
float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
)
cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key))
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@@ -149,7 +181,6 @@ def _get_token_base_cost(
1000 if "k" in threshold_str else 1
)
if usage.prompt_tokens > threshold:
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, key, prompt_base_cost)
)
@@ -194,7 +225,13 @@ def _get_token_base_cost(
except Exception:
continue
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
return (
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
)
def calculate_cost_component(
@@ -238,11 +275,227 @@ def _get_cost_per_unit(
verbose_logger.exception(
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
)
# If the service tier key doesn't exist or is None, try to fall back to the standard key
if cost_per_unit is None:
# Check if any service tier suffix exists in the cost key using ServiceTier enum
for service_tier in ServiceTier:
suffix = f"_{service_tier.value}"
if suffix in cost_key:
# Extract the base key by removing the matched suffix
base_key = cost_key.replace(suffix, "")
fallback_cost = model_info.get(base_key)
if isinstance(fallback_cost, float):
return fallback_cost
if isinstance(fallback_cost, int):
return float(fallback_cost)
if isinstance(fallback_cost, str):
try:
return float(fallback_cost)
except ValueError:
verbose_logger.exception(
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0"
)
break # Only try the first matching suffix
return default_value
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: Optional[CacheCreationTokenDetails],
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m = (
cache_creation_token_details.ephemeral_5m_input_tokens
)
cache_creation_tokens_1h = (
cache_creation_token_details.ephemeral_1h_input_tokens
)
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += (
cache_creation_tokens_5m * cache_creation_cost
if cache_creation_tokens_5m is not None
else 0.0
)
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr
if cache_creation_tokens_1h is not None
else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_creation_tokens: int
cache_creation_token_details: Optional[CacheCreationTokenDetails]
text_tokens: int
audio_tokens: int
character_count: int
image_count: int
video_length_seconds: int
def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cache_hit_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
or 0
)
cache_creation_tokens = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
)
or 0
)
cache_creation_token_details = (
cast(
Optional[CacheCreationTokenDetails],
getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
)
or None
)
text_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
character_count=character_count,
image_count=image_count,
video_length_seconds=video_length_seconds,
)
class CompletionTokensDetailsResult(TypedDict):
audio_tokens: int
text_tokens: int
reasoning_tokens: int
def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
return CompletionTokensDetailsResult(
audio_tokens=audio_tokens,
text_tokens=text_tokens,
reasoning_tokens=reasoning_tokens,
)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
"""
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details[
"cache_creation_token_details"
],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
)
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", prompt_tokens_details["character_count"]
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_video_per_second",
prompt_tokens_details["video_length_seconds"],
)
return prompt_cost
def generic_cost_per_token(
model: str, usage: Usage, custom_llm_provider: str
model: str,
usage: Usage,
custom_llm_provider: str,
service_tier: Optional[str] = None,
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -264,97 +517,47 @@ def generic_cost_per_token(
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
prompt_cost = 0.0
### PROCESSING COST
text_tokens = usage.prompt_tokens
cache_hit_tokens = 0
cache_creation_tokens = 0
audio_tokens = 0
character_count = 0
image_count = 0
video_length_seconds = 0
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0,
)
if usage.prompt_tokens_details:
cache_hit_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0)
)
or 0
)
cache_creation_tokens = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None)
)
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0))
or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
prompt_tokens_details = _parse_prompt_tokens_details(usage)
## EDGE CASE - text tokens not set inside PromptTokensDetails
if text_tokens == 0:
if prompt_tokens_details["text_tokens"] == 0:
text_tokens = (
usage.prompt_tokens
- cache_hit_tokens
- audio_tokens
- cache_creation_tokens
- prompt_tokens_details["cache_hit_tokens"]
- prompt_tokens_details["audio_tokens"]
- prompt_tokens_details["cache_creation_tokens"]
)
prompt_tokens_details["text_tokens"] = text_tokens
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = (
_get_token_base_cost(model_info=model_info, usage=usage)
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(
model_info=model_info, usage=usage, service_tier=service_tier
)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(cache_creation_tokens) * cache_creation_cost
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", character_count
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", image_count
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_video_per_second", video_length_seconds
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
prompt_base_cost=prompt_base_cost,
cache_read_cost=cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
)
## CALCULATE OUTPUT COST
@@ -363,27 +566,10 @@ def generic_cost_per_token(
reasoning_tokens = 0
is_text_tokens_total = False
if usage.completion_tokens_details is not None:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
completion_tokens_details = _parse_completion_tokens_details(usage)
audio_tokens = completion_tokens_details["audio_tokens"]
text_tokens = completion_tokens_details["text_tokens"]
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
if text_tokens == 0:
text_tokens = usage.completion_tokens
litellm/llms/anthropic/chat/transformation.py
+17 -1
View File
@@ -45,7 +45,10 @@ from litellm.types.llms.openai import (
OpenAIMcpServerTool,
OpenAIWebSearchOptions,
)
from litellm.types.utils import CompletionTokensDetailsWrapper
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.utils import (
@@ -820,6 +823,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
web_search_requests: Optional[int] = None
if (
"cache_creation_input_tokens" in _usage
@@ -842,8 +846,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
int, _usage["server_tool_use"]["web_search_requests"]
)
if "cache_creation" in _usage and _usage["cache_creation"] is not None:
cache_creation_token_details = CacheCreationTokenDetails(
ephemeral_5m_input_tokens=_usage["cache_creation"].get(
"ephemeral_5m_input_tokens"
),
ephemeral_1h_input_tokens=_usage["cache_creation"].get(
"ephemeral_1h_input_tokens"
),
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_read_input_tokens,
cache_creation_token_details=cache_creation_token_details,
)
completion_token_details = (
CompletionTokensDetailsWrapper(
litellm/llms/azure_ai/image_edit/__init__.py
+15
View File
@@ -0,0 +1,15 @@
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .transformation import AzureFoundryFluxImageEditConfig
__all__ = ["AzureFoundryFluxImageEditConfig"]
def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig:
model = model.lower()
model = model.replace("-", "")
model = model.replace("_", "")
if model == "" or "flux" in model: # empty model is flux
return AzureFoundryFluxImageEditConfig()
else:
raise ValueError(f"Model {model} is not supported for Azure AI image editing.")
litellm/llms/azure_ai/image_edit/transformation.py
+99
View File
@@ -0,0 +1,99 @@
from typing import Optional
import httpx
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.utils import _add_path_to_api_base
class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
"""
Azure AI Foundry FLUX image edit config
Supports FLUX models including FLUX-1-kontext-pro for image editing.
Azure AI Foundry FLUX models handle image editing through the /images/edits endpoint,
same as standard Azure OpenAI models. The request format uses multipart/form-data
with image files and prompt.
"""
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
"""
Validate Azure AI Foundry environment and set up authentication
Uses Api-Key header format
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key, # Azure AI Foundry uses Api-Key header format
}
)
return headers
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Constructs a complete URL for Azure AI Foundry image edits API request.
Azure AI Foundry FLUX models handle image editing through the /images/edits
endpoint.
Args:
- model: Model name (deployment name for Azure AI Foundry)
- api_base: Base URL for Azure AI endpoint
- litellm_params: Additional parameters including api_version
Returns:
- Complete URL for the image edits endpoint
"""
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter."
)
api_version = (litellm_params.get("api_version") or litellm.api_version
or get_secret_str("AZURE_AI_API_VERSION")
)
if api_version is None:
# API version is mandatory for Azure AI Foundry
raise ValueError(
"Azure API version is required. Set AZURE_AI_API_VERSION environment variable or pass api_version parameter."
)
# Add the path to the base URL using the model as deployment name
# Azure AI Foundry FLUX models use /images/edits for editing
if "/openai/deployments/" in api_base:
new_url = _add_path_to_api_base(
api_base=api_base,
ending_path="/images/edits",
)
else:
new_url = _add_path_to_api_base(
api_base=api_base,
ending_path=f"/openai/deployments/{model}/images/edits",
)
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params={"api-version": api_version})
return str(final_url)
litellm/llms/bedrock/chat/converse_transformation.py
+153 -45
View File
@@ -175,6 +175,77 @@ class AmazonConverseConfig(BaseConfig):
and v is not None
}
def _validate_request_metadata(self, metadata: dict) -> None:
"""
Validate requestMetadata according to AWS Bedrock Converse API constraints.
Constraints:
- Maximum of 16 items
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
"""
import re
if not isinstance(metadata, dict):
raise litellm.exceptions.BadRequestError(
message="requestMetadata must be a dictionary",
model="bedrock",
llm_provider="bedrock",
)
if len(metadata) > 16:
raise litellm.exceptions.BadRequestError(
message="requestMetadata can contain a maximum of 16 items",
model="bedrock",
llm_provider="bedrock",
)
key_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
value_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
for key, value in metadata.items():
if not isinstance(key, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata keys must be strings",
model="bedrock",
llm_provider="bedrock",
)
if not isinstance(value, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata values must be strings",
model="bedrock",
llm_provider="bedrock",
)
if len(key) == 0 or len(key) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata key length must be 1-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if len(value) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata value length must be 0-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if not key_pattern.match(key):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
if not value_pattern.match(value):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
def get_supported_openai_params(self, model: str) -> List[str]:
from litellm.utils import supports_function_calling
@@ -188,6 +259,7 @@ class AmazonConverseConfig(BaseConfig):
"top_p",
"extra_headers",
"response_format",
"requestMetadata",
]
if (
@@ -497,6 +569,10 @@ class AmazonConverseConfig(BaseConfig):
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
value
)
if param == "requestMetadata":
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value) # type: ignore
optional_params["requestMetadata"] = value
# Only update thinking tokens for non-GPT-OSS models
if "gpt-oss" not in model:
@@ -686,34 +762,10 @@ class AmazonConverseConfig(BaseConfig):
return {}
def _transform_request_helper(
self,
model: str,
system_content_blocks: List[SystemContentBlock],
optional_params: dict,
messages: Optional[List[AllMessageValues]] = None,
headers: Optional[dict] = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
Bedrock doesn't support tool calling without `tools=` param specified.
"""
if (
"tools" not in optional_params
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params:
optional_params["tools"] = add_dummy_tool(
custom_llm_provider="bedrock_converse"
)
else:
raise litellm.UnsupportedParamsError(
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
model="",
llm_provider="bedrock",
)
def _prepare_request_params(
self, optional_params: dict, model: str
) -> Tuple[dict, dict, dict]:
"""Prepare and separate request parameters."""
inference_params = copy.deepcopy(optional_params)
supported_converse_params = list(
AmazonConverseConfig.__annotations__.keys()
@@ -727,6 +779,11 @@ class AmazonConverseConfig(BaseConfig):
)
inference_params.pop("json_mode", None) # used for handling json_schema
# Extract requestMetadata before processing other parameters
request_metadata = inference_params.pop("requestMetadata", None)
if request_metadata is not None:
self._validate_request_metadata(request_metadata)
# keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
additional_request_params = {
k: v for k, v in inference_params.items() if k not in total_supported_params
@@ -740,9 +797,16 @@ class AmazonConverseConfig(BaseConfig):
self._handle_top_k_value(model, inference_params)
)
original_tools = inference_params.pop("tools", [])
return inference_params, additional_request_params, request_metadata
# Initialize bedrock_tools
def _process_tools_and_beta(
self,
original_tools: list,
model: str,
headers: Optional[dict],
additional_request_params: dict,
) -> Tuple[List[ToolBlock], list]:
"""Process tools and collect anthropic_beta values."""
bedrock_tools: List[ToolBlock] = []
# Collect anthropic_beta values from user headers
@@ -784,6 +848,48 @@ class AmazonConverseConfig(BaseConfig):
seen.add(beta)
additional_request_params["anthropic_beta"] = unique_betas
return bedrock_tools, anthropic_beta_list
def _transform_request_helper(
self,
model: str,
system_content_blocks: List[SystemContentBlock],
optional_params: dict,
messages: Optional[List[AllMessageValues]] = None,
headers: Optional[dict] = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
Bedrock doesn't support tool calling without `tools=` param specified.
"""
if (
"tools" not in optional_params
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params:
optional_params["tools"] = add_dummy_tool(
custom_llm_provider="bedrock_converse"
)
else:
raise litellm.UnsupportedParamsError(
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
model="",
llm_provider="bedrock",
)
# Prepare and separate parameters
inference_params, additional_request_params, request_metadata = (
self._prepare_request_params(optional_params, model)
)
original_tools = inference_params.pop("tools", [])
# Process tools and collect beta values
bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
original_tools, model, headers, additional_request_params
)
bedrock_tool_config: Optional[ToolConfigBlock] = None
if len(bedrock_tools) > 0:
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
@@ -813,6 +919,10 @@ class AmazonConverseConfig(BaseConfig):
if bedrock_tool_config is not None:
data["toolConfig"] = bedrock_tool_config
# Request Metadata (top-level field)
if request_metadata is not None:
data["requestMetadata"] = request_metadata
return data
async def _async_transform_request(
@@ -1059,9 +1169,7 @@ class AmazonConverseConfig(BaseConfig):
return message, returned_finish_reason
def _translate_message_content(
self, content_blocks: List[ContentBlock]
) -> Tuple[
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
@@ -1076,9 +1184,9 @@ class AmazonConverseConfig(BaseConfig):
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
for idx, content in enumerate(content_blocks):
"""
- Content is either a tool response or text
@@ -1199,9 +1307,9 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
if message is not None:
(
@@ -1214,12 +1322,12 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message["provider_specific_fields"] = {
"reasoningContentBlocks": reasoningContentBlocks,
}
chat_completion_message[
"reasoning_content"
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)
chat_completion_message["thinking_blocks"] = (
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message["content"] = content_str
if (
json_mode is True
litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py
+37 -15
View File
@@ -10,7 +10,7 @@ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-tit
"""
import types
from typing import List, Optional
from typing import List, Optional, Union
from litellm.types.llms.bedrock import (
AmazonTitanV2EmbeddingRequest,
@@ -30,9 +30,7 @@ class AmazonTitanV2Config:
normalize: Optional[bool] = None
dimensions: Optional[int] = None
def __init__(
self, normalize: Optional[bool] = None, dimensions: Optional[int] = None
) -> None:
def __init__(self, normalize: Optional[bool] = None, dimensions: Optional[int] = None) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
@@ -57,32 +55,56 @@ class AmazonTitanV2Config:
}
def get_supported_openai_params(self) -> List[str]:
return ["dimensions"]
return ["dimensions", "encoding_format"]
def map_openai_params(
self, non_default_params: dict, optional_params: dict
) -> dict:
def map_openai_params(self, non_default_params: dict, optional_params: dict) -> dict:
for k, v in non_default_params.items():
if k == "dimensions":
optional_params["dimensions"] = v
elif k == "encoding_format":
# Map OpenAI encoding_format to AWS embeddingTypes
if v == "float":
optional_params["embeddingTypes"] = ["float"]
elif v == "base64":
# base64 maps to binary format in AWS
optional_params["embeddingTypes"] = ["binary"]
else:
# For any other encoding format, default to float
optional_params["embeddingTypes"] = ["float"]
return optional_params
def _transform_request(
self, input: str, inference_params: dict
) -> AmazonTitanV2EmbeddingRequest:
def _transform_request(self, input: str, inference_params: dict) -> AmazonTitanV2EmbeddingRequest:
return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params) # type: ignore
def _transform_response(
self, response_list: List[dict], model: str
) -> EmbeddingResponse:
def _transform_response(self, response_list: List[dict], model: str) -> EmbeddingResponse:
total_prompt_tokens = 0
transformed_responses: List[Embedding] = []
for index, response in enumerate(response_list):
_parsed_response = AmazonTitanV2EmbeddingResponse(**response) # type: ignore
# According to AWS docs, embeddingsByType is always present
# If binary was requested (encoding_format="base64"), use binary data
# Otherwise, use float data from embeddingsByType or fallback to embedding field
embedding_data: Union[List[float], List[int]]
if ("embeddingsByType" in _parsed_response and
"binary" in _parsed_response["embeddingsByType"]):
# Use binary data if available (for encoding_format="base64")
embedding_data = _parsed_response["embeddingsByType"]["binary"]
elif ("embeddingsByType" in _parsed_response and
"float" in _parsed_response["embeddingsByType"]):
# Use float data from embeddingsByType
embedding_data = _parsed_response["embeddingsByType"]["float"]
elif "embedding" in _parsed_response:
# Fallback to legacy embedding field
embedding_data = _parsed_response["embedding"]
else:
raise ValueError(f"No embedding data found in response: {response}")
transformed_responses.append(
Embedding(
embedding=_parsed_response["embedding"],
embedding=embedding_data,
index=index,
object="embedding",
)
litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py
+7 -2
View File
@@ -7,12 +7,12 @@ from litellm.types.llms.bedrock import (
AmazonNovaCanvasColorGuidedGenerationParams,
AmazonNovaCanvasColorGuidedRequest,
AmazonNovaCanvasImageGenerationConfig,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
AmazonNovaCanvasRequestBase,
AmazonNovaCanvasTextToImageParams,
AmazonNovaCanvasTextToImageRequest,
AmazonNovaCanvasTextToImageResponse,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
)
from litellm.types.utils import ImageResponse
@@ -67,6 +67,11 @@ class AmazonNovaCanvasConfig:
"""
task_type = optional_params.pop("taskType", "TEXT_IMAGE")
image_generation_config = optional_params.pop("imageGenerationConfig", {})
# Extract model_id parameter to prevent "extraneous key" error from Bedrock API
# Following the same pattern as chat completions and embeddings
unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
image_generation_config = {**image_generation_config, **optional_params}
if task_type == "TEXT_IMAGE":
text_to_image_params: Dict[str, Any] = image_generation_config.pop(
litellm/llms/bedrock/image/image_handler.py
+11 -1
View File
@@ -233,7 +233,17 @@ class BedrockImageGeneration(BaseAWSLLM):
Returns:
dict: The request body to use for the Bedrock Image Generation API
"""
provider = model.split(".")[0]
# Use the existing ARN-aware provider detection method
bedrock_provider = self.get_bedrock_invoke_provider(model)
if bedrock_provider == "amazon" or bedrock_provider == "nova":
# Handle Amazon Nova Canvas models
provider = "amazon"
elif bedrock_provider == "stability":
provider = "stability"
else:
# Fallback to original logic for backward compatibility
provider = model.split(".")[0]
inference_params = copy.deepcopy(optional_params)
inference_params.pop(
"user", None
litellm/llms/gemini/google_genai/transformation.py
+2 -1
View File
@@ -121,7 +121,8 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
default_headers = {
"Content-Type": "application/json",
}
gemini_api_key = self._get_google_ai_studio_api_key(dict(litellm_params or {}))
# Use the passed api_key first, then fall back to litellm_params and environment
gemini_api_key = api_key or self._get_google_ai_studio_api_key(dict(litellm_params or {}))
if gemini_api_key is not None:
default_headers[self.XGOOGLE_API_KEY] = gemini_api_key
if headers is not None:
litellm/llms/gemini/image_generation/transformation.py
+78 -37
View File
@@ -85,17 +85,25 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
) -> str:
"""
Get the complete url for the request
Google AI API format: https://generativelanguage.googleapis.com/v1beta/models/{model}:predict
Gemini 2.5 Flash Image Preview: :generateContent
Other Imagen models: :predict
"""
complete_url: str = (
api_base
or get_secret_str("GEMINI_API_BASE")
api_base
or get_secret_str("GEMINI_API_BASE")
or self.DEFAULT_BASE_URL
)
complete_url = complete_url.rstrip("/")
complete_url = f"{complete_url}/models/{model}:predict"
# Gemini 2.5 Flash Image Preview uses generateContent endpoint
if "2.5-flash-image-preview" in model:
complete_url = f"{complete_url}/models/{model}:generateContent"
else:
# All other Imagen models use predict endpoint
complete_url = f"{complete_url}/models/{model}:predict"
return complete_url
def validate_environment(
@@ -128,35 +136,52 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
headers: dict,
) -> dict:
"""
Transform the image generation request to Google AI Imagen format
Google AI API format:
Transform the image generation request to Gemini format
For Gemini 2.5 Flash Image Preview, use the standard Gemini format with response_modalities:
{
"instances": [
"contents": [
{
"prompt": "Robot holding a red skateboard"
"parts": [
{"text": "Generate an image of..."}
]
}
],
"parameters": {
"sampleCount": 4,
"aspectRatio": "1:1",
"personGeneration": "allow_adult"
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
"""
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body.model_dump(exclude_none=True)
# For Gemini 2.5 Flash Image Preview, use standard Gemini format
if "2.5-flash-image-preview" in model:
request_body: dict = {
"contents": [
{
"parts": [
{"text": prompt}
]
}
],
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
return request_body
else:
# For other Imagen models, use the original Imagen format
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body_obj: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body_obj.model_dump(exclude_none=True)
def transform_image_generation_response(
self,
@@ -185,14 +210,30 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
if not model_response.data:
model_response.data = []
# Google AI returns predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
# Handle different response formats based on model
if "2.5-flash-image-preview" in model:
# Gemini 2.5 Flash Image Preview returns in candidates format
candidates = response_data.get("candidates", [])
for candidate in candidates:
content = candidate.get("content", {})
parts = content.get("parts", [])
for part in parts:
# Look for inlineData with image
if "inlineData" in part:
inline_data = part["inlineData"]
if "data" in inline_data:
model_response.data.append(ImageObject(
b64_json=inline_data["data"],
url=None,
))
else:
# Original Imagen format - predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
return model_response
litellm/llms/openai/cost_calculation.py
+7 -2
View File
@@ -18,7 +18,9 @@ def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_sec
return "cost_per_token"
def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
def cost_per_token(
model: str, usage: Usage, service_tier: Optional[str] = None
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -31,7 +33,10 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
"""
## CALCULATE INPUT COST
return generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="openai"
model=model,
usage=usage,
custom_llm_provider="openai",
service_tier=service_tier,
)
# ### Non-cached text tokens
# non_cached_text_tokens = usage.prompt_tokens
litellm/llms/vertex_ai/batches/transformation.py
+8 -1
View File
@@ -114,7 +114,14 @@ class VertexAIBatchTransformation:
"""
Gets the output file id from the Vertex AI Batch response
"""
output_file_id: str = ""
output_file_id: str = (
response.get("outputInfo", OutputInfo()).get("gcsOutputDirectory", "")
+ "/predictions.jsonl"
)
if output_file_id != "/predictions.jsonl":
return output_file_id
output_config = response.get("outputConfig")
if output_config is None:
return output_file_id
litellm/llms/vertex_ai/files/handler.py
+141 -2
View File
@@ -1,5 +1,6 @@
import asyncio
from typing import Any, Coroutine, Optional, Union
import urllib.parse
from typing import Any, Coroutine, Optional, Tuple, Union
import httpx
@@ -9,7 +10,12 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import (
GCSLoggingConfig,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import CreateFileRequest, OpenAIFileObject
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAIFileObject,
)
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from .transformation import VertexAIJsonlFilesTransformation
@@ -105,3 +111,136 @@ class VertexAIFilesHandler(GCSBucketBase):
max_retries=max_retries,
)
)
def _extract_bucket_and_object_from_file_id(self, file_id: str) -> Tuple[str, str]:
"""
Extract bucket name and object path from URL-encoded file_id.
Expected format: gs%3A%2F%2Fbucket-name%2Fpath%2Fto%2Ffile
Which decodes to: gs://bucket-name/path/to/file
Returns:
tuple: (bucket_name, url_encoded_object_path)
- bucket_name: "bucket-name"
- url_encoded_object_path: "path%2Fto%2Ffile"
"""
decoded_path = urllib.parse.unquote(file_id)
if decoded_path.startswith("gs://"):
full_path = decoded_path[5:] # Remove 'gs://' prefix
else:
full_path = decoded_path
if "/" in full_path:
bucket_name, object_path = full_path.split("/", 1)
else:
bucket_name = full_path
object_path = ""
encoded_object_path = urllib.parse.quote(object_path, safe="")
return bucket_name, encoded_object_path
async def afile_content(
self,
file_content_request: FileContentRequest,
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> HttpxBinaryResponseContent:
"""
Download file content from GCS bucket for VertexAI files.
Args:
file_content_request: Contains file_id (URL-encoded GCS path)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
"""
file_id = file_content_request.get("file_id")
if not file_id:
raise ValueError("file_id is required in file_content_request")
bucket_name, encoded_object_path = self._extract_bucket_and_object_from_file_id(
file_id
)
download_kwargs = {
"standard_callback_dynamic_params": {"gcs_bucket_name": bucket_name}
}
file_content = await self.download_gcs_object(
object_name=encoded_object_path, **download_kwargs
)
if file_content is None:
decoded_path = urllib.parse.unquote(file_id)
raise ValueError(f"Failed to download file from GCS: {decoded_path}")
decoded_path = urllib.parse.unquote(file_id)
mock_response = httpx.Response(
status_code=200,
content=file_content,
headers={"content-type": "application/octet-stream"},
request=httpx.Request(method="GET", url=decoded_path),
)
return HttpxBinaryResponseContent(response=mock_response)
def file_content(
self,
_is_async: bool,
file_content_request: FileContentRequest,
api_base: Optional[str],
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> Union[
HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
]:
"""
Download file content from GCS bucket for VertexAI files.
Supports both sync and async operations.
Args:
_is_async: Whether to run asynchronously
file_content_request: Contains file_id (URL-encoded GCS path)
api_base: API base (unused for GCS operations)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
"""
if _is_async:
return self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
else:
return asyncio.run(
self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
)
litellm/llms/vertex_ai/files/transformation.py
+3 -3
View File
@@ -261,10 +261,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
extracted_file_data = extract_file_data(file_data)
extracted_file_data_content = extracted_file_data.get("content")
if extracted_file_data_content is None:
raise ValueError("file content is required")
if FilesAPIUtils.is_batch_jsonl_file(
create_file_data=create_file_data,
extracted_file_data=extracted_file_data,
@@ -283,7 +283,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
openai_jsonl_content
)
)
return json.dumps(vertex_jsonl_content)
return "\n".join(json.dumps(item) for item in vertex_jsonl_content)
elif isinstance(extracted_file_data_content, bytes):
return extracted_file_data_content
else:
litellm/llms/vllm/common_utils.py
+18 -3
View File
@@ -11,7 +11,21 @@ from litellm.utils import _add_path_to_api_base
class VLLMError(BaseLLMException):
pass
def __init__(
self,
status_code: int,
message: str,
request: Optional[httpx.Request] = None,
response: Optional[httpx.Response] = None,
headers: Optional[Union[httpx.Headers, dict]] = None,
):
super().__init__(
status_code=status_code,
message=message,
request=request,
response=response,
headers=headers,
)
class VLLMModelInfo(BaseLLMModelInfo):
@@ -25,7 +39,8 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""Google AI Studio sends api key in query params"""
if api_key is not None:
headers["x-api-key"] = api_key
return headers
@staticmethod
@@ -53,7 +68,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
endpoint = "/v1/models"
if api_base is None or api_key is None:
raise ValueError(
"GEMINI_API_BASE or GEMINI_API_KEY is not set. Please set the environment variable, to query Gemini's `/models` endpoint."
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
)
url = _add_path_to_api_base(api_base, endpoint)
litellm/llms/wandb/__init__.py
View File
litellm/llms/wandb/chat/__init__.py
View File
litellm/llms/wandb/chat/transformation.py
+27
View File
@@ -0,0 +1,27 @@
"""
Wandb Chat Completions API - Transformation
This is OpenAI compatible - no translation needed / occurs
"""
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
class WandbConfig(OpenAIGPTConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
map max_completion_tokens param to max_tokens
"""
supported_openai_params = self.get_supported_openai_params(model=model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params
litellm/main.py
+39 -21
View File
@@ -1023,7 +1023,15 @@ def completion( # type: ignore # noqa: PLR0915
provider_specific_header = cast(
Optional[ProviderSpecificHeader], kwargs.get("provider_specific_header", None)
)
headers = kwargs.get("headers", None) or extra_headers
# Properly merge headers with priority: request headers > extra_headers > global litellm.headers
headers = {}
if litellm.headers is not None and isinstance(litellm.headers, dict):
headers.update(litellm.headers)
if extra_headers is not None and isinstance(extra_headers, dict):
headers.update(extra_headers)
request_headers = kwargs.get("headers", None)
if request_headers is not None and isinstance(request_headers, dict):
headers.update(request_headers)
ensure_alternating_roles: Optional[bool] = kwargs.get(
"ensure_alternating_roles", None
@@ -1034,10 +1042,6 @@ def completion( # type: ignore # noqa: PLR0915
assistant_continue_message: Optional[ChatCompletionAssistantMessage] = kwargs.get(
"assistant_continue_message", None
)
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
num_retries = kwargs.get(
"num_retries", None
) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor.
@@ -1446,8 +1450,7 @@ def completion( # type: ignore # noqa: PLR0915
"azure_ad_token_provider", None
)
headers = headers or litellm.headers
# Use the consolidated headers that were already merged at the top of the function
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
if max_retries is not None:
@@ -1714,8 +1717,7 @@ def completion( # type: ignore # noqa: PLR0915
or get_secret("OPENAI_API_KEY")
)
headers = headers or litellm.headers
# Use the consolidated headers that were already merged at the top of the function
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
@@ -2005,6 +2007,7 @@ def completion( # type: ignore # noqa: PLR0915
or custom_llm_provider == "openai"
or custom_llm_provider == "together_ai"
or custom_llm_provider == "nebius"
or custom_llm_provider == "wandb"
or custom_llm_provider in litellm.openai_compatible_providers
or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo
): # allow user to make an openai call with a custom base
@@ -2440,12 +2443,8 @@ def completion( # type: ignore # noqa: PLR0915
or "https://api.cohere.ai/v1/chat"
)
headers = headers or litellm.headers or {}
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
# Use the consolidated headers that were already merged at the top of the function
# No need for additional merging here as it's already done
response = base_llm_http_handler.completion(
model=model,
@@ -3843,7 +3842,7 @@ def embedding(
*,
aembedding: Literal[True],
**kwargs,
) -> Coroutine[Any, Any, EmbeddingResponse]:
) -> Coroutine[Any, Any, EmbeddingResponse]:
...
@@ -3869,7 +3868,7 @@ def embedding(
*,
aembedding: Literal[False] = False,
**kwargs,
) -> EmbeddingResponse:
) -> EmbeddingResponse:
...
# fmt: on
@@ -4180,10 +4179,8 @@ def embedding( # noqa: PLR0915
or litellm.api_key
)
if extra_headers is not None and isinstance(extra_headers, dict):
headers = extra_headers
else:
headers = {}
# Use the consolidated headers that were already merged at the top of the function
# No need for additional merging here as it's already done
response = base_llm_http_handler.embedding(
model=model,
@@ -4445,6 +4442,27 @@ def embedding( # noqa: PLR0915
or "api.studio.nebius.ai/v1"
)
response = openai_chat_completions.embedding(
model=model,
input=input,
api_base=api_base,
api_key=api_key,
logging_obj=logging,
timeout=timeout,
model_response=EmbeddingResponse(),
optional_params=optional_params,
client=client,
aembedding=aembedding,
)
elif custom_llm_provider == "wandb":
api_key = api_key or litellm.api_key or get_secret_str("WANDB_API_KEY")
api_base = (
api_base
or litellm.api_base
or get_secret_str("WANDB_API_BASE")
or "https://api.inference.wandb.ai/v1"
)
response = openai_chat_completions.embedding(
model=model,
input=input,
litellm/model_prices_and_context_window_backup.json
+202 -3
View File
@@ -9126,7 +9126,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
@@ -10489,7 +10489,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
@@ -11534,8 +11534,10 @@
},
"gpt-4.1": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@@ -11543,6 +11545,7 @@
"mode": "chat",
"output_cost_per_token": 8e-06,
"output_cost_per_token_batches": 4e-06,
"output_cost_per_token_priority": 1.4e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -11600,8 +11603,10 @@
},
"gpt-4.1-mini": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_priority": 1.75e-07,
"input_cost_per_token": 4e-07,
"input_cost_per_token_batches": 2e-07,
"input_cost_per_token_priority": 7e-07,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@@ -11609,6 +11614,7 @@
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"output_cost_per_token_batches": 8e-07,
"output_cost_per_token_priority": 2.8e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -11666,8 +11672,10 @@
},
"gpt-4.1-nano": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_priority": 5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
"input_cost_per_token_priority": 2e-07,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@@ -11675,6 +11683,7 @@
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_batches": 2e-07,
"output_cost_per_token_priority": 8e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -11773,8 +11782,10 @@
},
"gpt-4o": {
"cache_read_input_token_cost": 1.25e-06,
"cache_read_input_token_cost_priority": 2.125e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 4.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
@@ -11782,6 +11793,7 @@
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_priority": 1.7e-05,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@@ -11794,6 +11806,7 @@
"gpt-4o-2024-05-13": {
"input_cost_per_token": 5e-06,
"input_cost_per_token_batches": 2.5e-06,
"input_cost_per_token_priority": 8.75e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 4096,
@@ -11801,6 +11814,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.625e-05,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@@ -11919,8 +11933,10 @@
},
"gpt-4o-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_priority": 1.25e-07,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_priority": 2.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
@@ -11928,6 +11944,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"output_cost_per_token_batches": 3e-07,
"output_cost_per_token_priority": 1e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@@ -12243,13 +12260,19 @@
},
"gpt-5": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
"cache_read_input_token_cost_priority": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_flex": 5e-06,
"output_cost_per_token_priority": 2e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -12275,13 +12298,19 @@
},
"gpt-5-2025-08-07": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
"cache_read_input_token_cost_priority": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_flex": 5e-06,
"output_cost_per_token_priority": 2e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -12371,13 +12400,19 @@
},
"gpt-5-mini": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2e-06,
"output_cost_per_token_flex": 1e-06,
"output_cost_per_token_priority": 3.6e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -12403,13 +12438,19 @@
},
"gpt-5-mini-2025-08-07": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2e-06,
"output_cost_per_token_flex": 1e-06,
"output_cost_per_token_priority": 3.6e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -12435,13 +12476,16 @@
},
"gpt-5-nano": {
"cache_read_input_token_cost": 5e-09,
"cache_read_input_token_cost_flex": 2.5e-09,
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_flex": 2e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -12467,13 +12511,16 @@
},
"gpt-5-nano-2025-08-07": {
"cache_read_input_token_cost": 5e-09,
"cache_read_input_token_cost_flex": 2.5e-09,
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_flex": 2e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@@ -15177,13 +15224,19 @@
},
"o3": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_flex": 1e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 8e-06,
"output_cost_per_token_flex": 4e-06,
"output_cost_per_token_priority": 1.4e-05,
"supported_endpoints": [
"/v1/responses",
"/v1/chat/completions",
@@ -15399,13 +15452,19 @@
},
"o4-mini": {
"cache_read_input_token_cost": 2.75e-07,
"cache_read_input_token_cost_flex": 1.38e-07,
"cache_read_input_token_cost_priority": 5e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_flex": 5.5e-07,
"input_cost_per_token_priority": 2e-06,
"litellm_provider": "openai",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"output_cost_per_token_flex": 2.2e-06,
"output_cost_per_token_priority": 8e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_pdf_input": true,
@@ -16900,6 +16959,20 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"openrouter/x-ai/grok-4-fast:free": {
"input_cost_per_token": 0,
"litellm_provider": "openrouter",
"max_input_tokens": 2000000,
"max_output_tokens": 30000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 0,
"source": "https://openrouter.ai/x-ai/grok-4-fast:free",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": false
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",
@@ -20943,6 +21016,132 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"wandb/openai/gpt-oss-120b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.015,
"output_cost_per_token": 0.06,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.005,
"output_cost_per_token": 0.02,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/zai-org/GLM-4.5": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.055,
"output_cost_per_token": 0.2,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.01,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.1,
"output_cost_per_token": 0.15,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.01,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.135,
"output_cost_per_token": 0.4,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.022,
"output_cost_per_token": 0.022,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.055,
"output_cost_per_token": 0.165,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
"input_cost_per_token": 0.135,
"output_cost_per_token": 0.54,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
"input_cost_per_token": 0.114,
"output_cost_per_token": 0.275,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-3.3-70B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.071,
"output_cost_per_token": 0.071,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
"input_cost_per_token": 0.017,
"output_cost_per_token": 0.066,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.008,
"output_cost_per_token": 0.035,
"litellm_provider": "wandb",
"mode": "chat"
},
"watsonx/ibm/granite-3-8b-instruct": {
"input_cost_per_token": 0.0002,
"litellm_provider": "watsonx",
@@ -21337,4 +21536,4 @@
"supports_vision": true,
"supports_web_search": true
}
}
}
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+131 -74
View File
@@ -28,12 +28,16 @@ class MCPRequestHandler:
LITELLM_MCP_SERVERS_HEADER_NAME = SpecialHeaders.mcp_servers.value
LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value
# MCP Protocol Version header
MCP_PROTOCOL_VERSION_HEADER_NAME = "MCP-Protocol-Version"
@staticmethod
async def process_mcp_request(scope: Scope) -> Tuple[UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]]:
async def process_mcp_request(
scope: Scope,
) -> Tuple[
UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]]
]:
"""
Process and validate MCP request headers from the ASGI scope.
This includes:
@@ -49,7 +53,6 @@ class MCPRequestHandler:
mcp_auth_header: Optional[str] MCP auth header to be passed to the MCP server (deprecated)
mcp_servers: Optional[List[str]] List of MCP servers and access groups to use
mcp_server_auth_headers: Optional[Dict[str, str]] Server-specific auth headers in format {server_alias: auth_value}
mcp_protocol_version: Optional[str] MCP protocol version from request header
Raises:
HTTPException: If headers are invalid or missing required headers
@@ -58,39 +61,50 @@ class MCPRequestHandler:
litellm_api_key = (
MCPRequestHandler.get_litellm_api_key_from_headers(headers) or ""
)
# Get the old mcp_auth_header for backward compatibility
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
# Get the new server-specific auth headers
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
# Get MCP protocol version from header
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
# Get the new server-specific auth headers
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
# Parse MCP servers from header
mcp_servers_header = headers.get(MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME)
mcp_servers_header = headers.get(
MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME
)
verbose_logger.debug(f"Raw MCP servers header: {mcp_servers_header}")
mcp_servers = None
if mcp_servers_header is not None:
try:
mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()]
mcp_servers = [
s.strip() for s in mcp_servers_header.split(",") if s.strip()
]
verbose_logger.debug(f"Parsed MCP servers: {mcp_servers}")
except Exception as e:
verbose_logger.debug(f"Error parsing mcp_servers header: {e}")
mcp_servers = None
if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0):
if mcp_servers_header == "" or (
mcp_servers is not None and len(mcp_servers) == 0
):
mcp_servers = []
# Create a proper Request object with mock body method to avoid ASGI receive channel issues
request = Request(scope=scope)
async def mock_body():
return b"{}"
request.body = mock_body # type: ignore
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
return validated_user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return (
validated_user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
)
@staticmethod
def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]:
@@ -104,10 +118,12 @@ class MCPRequestHandler:
Support this auth: https://docs.litellm.ai/docs/mcp#using-your-mcp-with-client-side-credentials
If you want to use a different header name, you can set the `LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME` in the secret manager or `mcp_client_side_auth_header_name` in the general settings.
DEPRECATED: This method is deprecated in favor of server-specific auth headers using the format x-mcp-{{server_alias}}-{{header_name}} instead.
"""
mcp_client_side_auth_header_name: str = MCPRequestHandler._get_mcp_client_side_auth_header_name()
mcp_client_side_auth_header_name: str = (
MCPRequestHandler._get_mcp_client_side_auth_header_name()
)
auth_header = headers.get(mcp_client_side_auth_header_name)
if auth_header:
verbose_logger.warning(
@@ -115,42 +131,49 @@ class MCPRequestHandler:
f"Please use server-specific auth headers in the format 'x-mcp-{{server_alias}}-{{header_name}}' instead."
)
return auth_header
@staticmethod
def _get_mcp_server_auth_headers_from_headers(headers: Headers) -> Dict[str, str]:
"""
Parse server-specific MCP auth headers from the request headers.
Looks for headers in the format: x-mcp-{server_alias}-{header_name}
Examples:
- x-mcp-github-authorization: Bearer token123
- x-mcp-zapier-x-api-key: api_key_456
- x-mcp-deepwiki-authorization: Basic base64_encoded_creds
Returns:
Dict[str, str]: Mapping of server alias to auth value
"""
server_auth_headers = {}
prefix = "x-mcp-"
for header_name, header_value in headers.items():
if header_name.lower().startswith(prefix):
# Skip the access groups header as it's not a server auth header
if header_name.lower() == MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower() or header_name.lower() == MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower():
if (
header_name.lower()
== MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower()
or header_name.lower()
== MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower()
):
continue
# Extract server_alias and header_name from x-mcp-{server_alias}-{header_name}
remaining = header_name[len(prefix):].lower()
if '-' in remaining:
remaining = header_name[len(prefix) :].lower()
if "-" in remaining:
# Split on the last dash to separate server_alias from header_name
parts = remaining.rsplit('-', 1)
parts = remaining.rsplit("-", 1)
if len(parts) == 2:
server_alias, auth_header_name = parts
server_auth_headers[server_alias] = header_value
verbose_logger.debug(f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}...")
verbose_logger.debug(
f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}..."
)
return server_auth_headers
@staticmethod
def _get_mcp_client_side_auth_header_name() -> str:
"""
@@ -162,13 +185,21 @@ class MCPRequestHandler:
"""
from litellm.proxy.proxy_server import general_settings
from litellm.secret_managers.main import get_secret_str
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = general_settings.get("mcp_client_side_auth_header_name") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = (
MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
)
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
general_settings.get("mcp_client_side_auth_header_name")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
@staticmethod
def get_litellm_api_key_from_headers(headers: Headers) -> Optional[str]:
@@ -229,10 +260,14 @@ class MCPRequestHandler:
try:
allowed_mcp_servers: List[str] = []
allowed_mcp_servers_for_key = (
await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_key(
user_api_key_auth
)
)
allowed_mcp_servers_for_team = (
await MCPRequestHandler._get_allowed_mcp_servers_for_team(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_team(
user_api_key_auth
)
)
#########################################################
@@ -274,7 +309,9 @@ class MCPRequestHandler:
try:
key_object_permission = (
await prisma_client.db.litellm_objectpermissiontable.find_unique(
where={"object_permission_id": user_api_key_auth.object_permission_id},
where={
"object_permission_id": user_api_key_auth.object_permission_id
},
)
)
if key_object_permission is None:
@@ -282,17 +319,21 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = key_object_permission.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for key: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for key: {str(e)}"
)
return []
@staticmethod
@@ -318,10 +359,10 @@ class MCPRequestHandler:
return []
try:
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@@ -333,21 +374,27 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = object_permissions.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for team: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for team: {str(e)}"
)
return []
@staticmethod
def _get_config_server_ids_for_access_groups(config_mcp_servers, access_groups: List[str]) -> Set[str]:
def _get_config_server_ids_for_access_groups(
config_mcp_servers, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from config-loaded servers that match any of the given access groups.
"""
@@ -359,7 +406,9 @@ class MCPRequestHandler:
return server_ids
@staticmethod
async def _get_db_server_ids_for_access_groups(prisma_client, access_groups: List[str]) -> Set[str]:
async def _get_db_server_ids_for_access_groups(
prisma_client, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from DB servers that match any of the given access groups.
"""
@@ -367,21 +416,19 @@ class MCPRequestHandler:
if access_groups and prisma_client is not None:
try:
mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many(
where={
"mcp_access_groups": {
"hasSome": access_groups
}
}
where={"mcp_access_groups": {"hasSome": access_groups}}
)
for server in mcp_servers:
server_ids.add(server.server_id)
except Exception as e:
verbose_logger.debug(f"Error getting MCP servers from access groups: {e}")
verbose_logger.debug(
f"Error getting MCP servers from access groups: {e}"
)
return server_ids
@staticmethod
async def _get_mcp_servers_from_access_groups(
access_groups: List[str]
access_groups: List[str],
) -> List[str]:
"""
Resolve MCP access groups to server IDs by querying BOTH the MCP server table (DB) AND config-loaded servers
@@ -390,22 +437,28 @@ class MCPRequestHandler:
try:
# Import here to avoid circular import
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
# Use the new helper for config-loaded servers
server_ids = MCPRequestHandler._get_config_server_ids_for_access_groups(
global_mcp_server_manager.config_mcp_servers, access_groups
)
# Use the new helper for DB servers
db_server_ids = await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
db_server_ids = (
await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
)
)
server_ids.update(db_server_ids)
return list(server_ids)
except Exception as e:
verbose_logger.warning(f"Failed to get MCP servers from access groups: {str(e)}")
verbose_logger.warning(
f"Failed to get MCP servers from access groups: {str(e)}"
)
return []
@staticmethod
@@ -418,8 +471,8 @@ class MCPRequestHandler:
from typing import List
access_groups: List[str] = []
access_groups_for_key = (
await MCPRequestHandler._get_mcp_access_groups_for_key(user_api_key_auth)
access_groups_for_key = await MCPRequestHandler._get_mcp_access_groups_for_key(
user_api_key_auth
)
access_groups_for_team = (
await MCPRequestHandler._get_mcp_access_groups_for_team(user_api_key_auth)
@@ -482,10 +535,10 @@ class MCPRequestHandler:
verbose_logger.debug("prisma_client is None")
return []
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@@ -502,10 +555,14 @@ class MCPRequestHandler:
"""
Extract and parse the x-mcp-access-groups header as a list of strings.
"""
mcp_access_groups_header = headers.get(MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME)
mcp_access_groups_header = headers.get(
MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME
)
if mcp_access_groups_header is not None:
try:
return [s.strip() for s in mcp_access_groups_header.split(",") if s.strip()]
return [
s.strip() for s in mcp_access_groups_header.split(",") if s.strip()
]
except Exception:
return None
return None
@@ -516,4 +573,4 @@ class MCPRequestHandler:
Extract and parse the x-mcp-access-groups header from an ASGI scope.
"""
headers = MCPRequestHandler._safe_get_headers_from_scope(scope)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+88 -113
View File
@@ -34,8 +34,6 @@ from litellm.proxy._experimental.mcp_server.utils import (
from litellm.proxy._types import (
LiteLLM_MCPServerTable,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPTransport,
MCPTransportType,
UserAPIKeyAuth,
@@ -70,38 +68,6 @@ def _deserialize_env_dict(env_data: Any) -> Optional[Dict[str, str]]:
return env_data
def _convert_protocol_version_to_enum(
protocol_version: Optional[str | MCPSpecVersionType],
) -> MCPSpecVersionType:
"""
Convert string protocol version to MCPSpecVersion enum.
Args:
protocol_version: String protocol version, enum, or None
Returns:
MCPSpecVersionType: The enum value
"""
if not protocol_version:
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
# If it's already an MCPSpecVersion enum, return it
if isinstance(protocol_version, MCPSpecVersion):
return cast(MCPSpecVersionType, protocol_version)
# If it's a string, try to match it to enum values
if isinstance(protocol_version, str):
for version in MCPSpecVersion:
if version.value == protocol_version:
return cast(MCPSpecVersionType, version)
# If no match found, return default
verbose_logger.warning(
f"Unknown protocol version '{protocol_version}', using default"
)
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
class MCPServerManager:
def __init__(self):
self.registry: Dict[str, MCPServer] = {}
@@ -113,8 +79,7 @@ class MCPServerManager:
"name": "zapier_mcp_server",
"url": "https://actions.zapier.com/mcp/sk-ak-2ew3bofIeQIkNoeKIdXrF1Hhhp/sse"
"transport": "sse",
"auth_type": "api_key",
"spec_version": "2025-03-26"
"auth_type": "api_key"
},
"uuid-2": {
"name": "google_drive_mcp_server",
@@ -156,15 +121,13 @@ class MCPServerManager:
for server_name, server_config in mcp_servers_config.items():
validate_mcp_server_name(server_name)
_mcp_info: Dict[str, Any] = server_config.get("mcp_info", None) or {}
# Convert Dict[str, Any] to MCPInfo properly
mcp_info: MCPInfo = {
"server_name": _mcp_info.get("server_name", server_name),
"description": _mcp_info.get(
"description", server_config.get("description", None)
),
"logo_url": _mcp_info.get("logo_url", None),
"mcp_server_cost_info": _mcp_info.get("mcp_server_cost_info", None),
}
# Preserve all custom fields from config while setting defaults for core fields
mcp_info: MCPInfo = _mcp_info.copy()
# Set default values for core fields if not present
if "server_name" not in mcp_info:
mcp_info["server_name"] = server_name
if "description" not in mcp_info and server_config.get("description"):
mcp_info["description"] = server_config.get("description")
# Use alias for name if present, else server_name
alias = server_config.get("alias", None)
@@ -223,7 +186,6 @@ class MCPServerManager:
server_name=server_name,
url=server_config.get("url", None) or "",
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
alias=alias,
)
@@ -239,7 +201,6 @@ class MCPServerManager:
env=server_config.get("env", None) or {},
# TODO: utility fn the default values
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
authentication_token=server_config.get(
"authentication_token", server_config.get("auth_value", None)
@@ -280,6 +241,14 @@ class MCPServerManager:
name_for_prefix = (
mcp_server.alias or mcp_server.server_name or mcp_server.server_id
)
# Preserve all custom fields from database while setting defaults for core fields
mcp_info: MCPInfo = _mcp_info.copy()
# Set default values for core fields if not present
if "server_name" not in mcp_info:
mcp_info["server_name"] = mcp_server.server_name or mcp_server.server_id
if "description" not in mcp_info and mcp_server.description:
mcp_info["description"] = mcp_server.description
new_server = MCPServer(
server_id=mcp_server.server_id,
name=name_for_prefix,
@@ -287,13 +256,8 @@ class MCPServerManager:
server_name=getattr(mcp_server, "server_name", None),
url=mcp_server.url,
transport=cast(MCPTransportType, mcp_server.transport),
spec_version=_convert_protocol_version_to_enum(mcp_server.spec_version),
auth_type=cast(MCPAuthType, mcp_server.auth_type),
mcp_info=MCPInfo(
server_name=mcp_server.server_name or mcp_server.server_id,
description=mcp_server.description,
mcp_server_cost_info=_mcp_info.get("mcp_server_cost_info", None),
),
mcp_info=mcp_info,
# Stdio-specific fields
command=getattr(mcp_server, "command", None),
args=getattr(mcp_server, "args", None) or [],
@@ -350,7 +314,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all tools available across all MCP Servers.
@@ -390,7 +353,6 @@ class MCPServerManager:
tools = await self._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
list_tools_result.extend(tools)
verbose_logger.info(
@@ -414,7 +376,6 @@ class MCPServerManager:
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
protocol_version: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@@ -422,18 +383,12 @@ class MCPServerManager:
Args:
server (MCPServer): The server configuration
mcp_auth_header: MCP auth header to be passed to the MCP server. This is optional and will be used if provided.
protocol_version: Optional MCP protocol version to use. If not provided, uses server's default.
Returns:
MCPClient: Configured MCP client instance
"""
transport = server.transport or MCPTransport.sse
# Convert protocol version string to enum
protocol_version_enum = _convert_protocol_version_to_enum(
protocol_version or server.spec_version
)
# Handle stdio transport
if transport == MCPTransport.stdio:
# For stdio, we need to get the stdio config from the server
@@ -450,7 +405,6 @@ class MCPServerManager:
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
stdio_config=stdio_config,
protocol_version=protocol_version_enum,
)
else:
# For HTTP/SSE transports
@@ -461,14 +415,13 @@ class MCPServerManager:
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
protocol_version=protocol_version_enum,
)
async def _get_tools_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
mcp_protocol_version: Optional[str] = None,
add_prefix: bool = True,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@@ -483,23 +436,21 @@ class MCPServerManager:
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"_get_tools_from_server for {server.name}...")
protocol_version = (
mcp_protocol_version if mcp_protocol_version else server.spec_version
)
client = None
try:
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
protocol_version=protocol_version,
)
tools = await self._fetch_tools_with_timeout(client, server.name)
prefixed_tools = self._create_prefixed_tools(tools, server)
return prefixed_tools
prefixed_or_original_tools = self._create_prefixed_tools(
tools, server, add_prefix=add_prefix
)
return prefixed_or_original_tools
except Exception as e:
verbose_logger.warning(
@@ -530,7 +481,7 @@ class MCPServerManager:
async def _list_tools_task():
try:
await client.connect()
tools = await client.list_tools()
verbose_logger.debug(f"Tools from {server_name}: {tools}")
return tools
@@ -568,7 +519,7 @@ class MCPServerManager:
return []
def _create_prefixed_tools(
self, tools: List[MCPTool], server: MCPServer
self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True
) -> List[MCPTool]:
"""
Create prefixed tools and update tool mapping.
@@ -586,14 +537,16 @@ class MCPServerManager:
for tool in tools:
prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix)
prefixed_tool = MCPTool(
name=prefixed_name,
name_to_use = prefixed_name if add_prefix else tool.name
tool_obj = MCPTool(
name=name_to_use,
description=tool.description,
inputSchema=tool.inputSchema,
)
prefixed_tools.append(prefixed_tool)
prefixed_tools.append(tool_obj)
# Update tool to server mapping with both original and prefixed names
# Update tool to server mapping for resolution (support both forms)
self.tool_name_to_mcp_server_name_mapping[tool.name] = prefix
self.tool_name_to_mcp_server_name_mapping[prefixed_name] = prefix
@@ -609,7 +562,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> CallToolResult:
"""
@@ -660,32 +612,54 @@ class MCPServerManager:
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
"user_api_key_user_id": getattr(user_api_key_auth, 'user_id', None) if user_api_key_auth else None,
"user_api_key_team_id": getattr(user_api_key_auth, 'team_id', None) if user_api_key_auth else None,
"user_api_key_end_user_id": getattr(user_api_key_auth, 'end_user_id', None) if user_api_key_auth else None,
"user_api_key_hash": getattr(user_api_key_auth, 'api_key_hash', None) if user_api_key_auth else None,
"user_api_key_user_id": getattr(user_api_key_auth, "user_id", None)
if user_api_key_auth
else None,
"user_api_key_team_id": getattr(user_api_key_auth, "team_id", None)
if user_api_key_auth
else None,
"user_api_key_end_user_id": getattr(
user_api_key_auth, "end_user_id", None
)
if user_api_key_auth
else None,
"user_api_key_hash": getattr(user_api_key_auth, "api_key_hash", None)
if user_api_key_auth
else None,
}
# Create MCP request object for processing
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(pre_hook_kwargs)
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(
pre_hook_kwargs
)
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
mcp_request_obj, pre_hook_kwargs
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, #type: ignore
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = proxy_logging_obj._convert_mcp_hook_response_to_kwargs(modified_data, pre_hook_kwargs)
modified_kwargs = (
proxy_logging_obj._convert_mcp_hook_response_to_kwargs(
modified_data, pre_hook_kwargs
)
)
if modified_kwargs.get("arguments") != arguments:
arguments = modified_kwargs["arguments"]
except (BlockedPiiEntityError, GuardrailRaisedException, HTTPException) as e:
except (
BlockedPiiEntityError,
GuardrailRaisedException,
HTTPException,
) as e:
# Re-raise guardrail exceptions to properly fail the MCP call
verbose_logger.error(
f"Guardrail blocked MCP tool call pre call: {str(e)}"
@@ -706,11 +680,9 @@ class MCPServerManager:
client = self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
protocol_version=mcp_protocol_version,
)
async with client:
# Use the original tool name (without prefix) for the actual call
call_tool_params = MCPCallToolRequestParams(
name=original_tool_name,
@@ -721,7 +693,7 @@ class MCPServerManager:
# Create synthetic LLM data for during hook processing
from litellm.types.llms.base import HiddenParams
from litellm.types.mcp import MCPDuringCallRequestObject
request_obj = MCPDuringCallRequestObject(
tool_name=name,
arguments=arguments,
@@ -729,28 +701,29 @@ class MCPServerManager:
start_time=start_time.timestamp() if start_time else None,
hidden_params=HiddenParams(),
)
during_hook_kwargs = {
"name": name,
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
}
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(request_obj, during_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
request_obj, during_hook_kwargs
)
during_hook_task = asyncio.create_task(
proxy_logging_obj.during_call_hook(
user_api_key_dict=user_api_key_auth,
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
)
tasks.append(during_hook_task)
tasks.append(asyncio.create_task(client.call_tool(call_tool_params)))
try:
mcp_responses = await asyncio.gather(*tasks)
# If proxy_logging_obj is None, the tool call result is at index 0
@@ -839,19 +812,21 @@ class MCPServerManager:
)
verbose_logger.info("Loading MCP servers from database into registry...")
# perform authz check to filter the mcp servers user has access to
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
db_mcp_servers = await get_all_mcp_servers(prisma_client)
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
# ensure the global_mcp_server_manager is up to date with the db
for server in db_mcp_servers:
verbose_logger.debug(f"Adding server to registry: {server.server_id} ({server.server_name})")
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
self.add_update_server(server)
verbose_logger.info(f"Registry now contains {len(self.get_registry())} servers")
def get_mcp_server_by_id(self, server_id: str) -> Optional[MCPServer]:
@@ -869,7 +844,6 @@ class MCPServerManager:
server_name: str,
url: str,
transport: str,
spec_version: str,
auth_type: Optional[str] = None,
alias: Optional[str] = None,
) -> str:
@@ -885,7 +859,6 @@ class MCPServerManager:
server_name: Name of the server
url: Server URL
transport: Transport type (sse, http, etc.)
spec_version: MCP spec version
auth_type: Authentication type (optional)
alias: Server alias (optional)
@@ -893,7 +866,9 @@ class MCPServerManager:
A deterministic server ID string
"""
# Create a string from all the identifying parameters
params_string = f"{server_name}|{url}|{transport}|{spec_version}|{auth_type or ''}|{alias or ''}"
params_string = (
f"{server_name}|{url}|{transport}|{auth_type or ''}|{alias or ''}"
)
# Generate SHA-256 hash
hash_object = hashlib.sha256(params_string.encode("utf-8"))
@@ -1050,11 +1025,12 @@ class MCPServerManager:
alias=_server_config.alias,
url=_server_config.url,
transport=_server_config.transport,
spec_version=_server_config.spec_version,
auth_type=_server_config.auth_type,
created_at=datetime.datetime.now(),
updated_at=datetime.datetime.now(),
description=_server_config.mcp_info.get("description") if _server_config.mcp_info else None,
description=_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None,
mcp_info=_server_config.mcp_info,
mcp_access_groups=_server_config.access_groups or [],
# Stdio-specific fields
@@ -1111,7 +1087,6 @@ class MCPServerManager:
description=server.description,
url=server.url,
transport=server.transport,
spec_version=server.spec_version,
auth_type=server.auth_type,
created_at=server.created_at,
created_by=server.created_by,
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+79 -50
View File
@@ -23,7 +23,6 @@ router = APIRouter(
if MCP_AVAILABLE:
from litellm.experimental_mcp_client.client import MCPTool
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
_convert_protocol_version_to_enum,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.server import (
@@ -34,18 +33,24 @@ if MCP_AVAILABLE:
########################################################
############ MCP Server REST API Routes #################
def _get_server_auth_header(
server, mcp_server_auth_headers: Optional[Dict[str, str]], mcp_auth_header: Optional[str]
server,
mcp_server_auth_headers: Optional[Dict[str, str]],
mcp_auth_header: Optional[str],
) -> Optional[str]:
"""Helper function to get server-specific auth header with case-insensitive matching."""
if mcp_server_auth_headers and server.alias:
normalized_server_alias = server.alias.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_alias)
if server_auth is not None:
return server_auth
elif mcp_server_auth_headers and server.server_name:
normalized_server_name = server.server_name.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_name)
if server_auth is not None:
return server_auth
@@ -63,12 +68,12 @@ if MCP_AVAILABLE:
for tool in tools
]
async def _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version):
async def _get_tools_for_single_server(server, server_auth_header):
"""Helper function to get tools for a single server."""
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
add_prefix=False,
)
return _create_tool_response_objects(tools, server.mcp_info)
@@ -104,17 +109,20 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
try:
# Extract auth headers from request
headers = request.headers
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(
headers
)
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
list_tools_result = []
error_message = None
# If server_id is specified, only query that specific server
if server_id:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
@@ -122,49 +130,67 @@ if MCP_AVAILABLE:
return {
"tools": [],
"error": "server_not_found",
"message": f"Server with id {server_id} not found"
"message": f"Server with id {server_id} not found",
}
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
list_tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
list_tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
return {
"tools": [],
"error": "server_error",
"message": f"Failed to get tools from server {server.name}: {str(e)}"
"message": f"Failed to get tools from server {server.name}: {str(e)}",
}
else:
# Query all servers
errors = []
for server in global_mcp_server_manager.get_registry().values():
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
list_tools_result.extend(tools_result)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
errors.append(f"{server.name}: {str(e)}")
continue
if errors and not list_tools_result:
error_message = "Failed to get tools from servers: " + "; ".join(errors)
error_message = "Failed to get tools from servers: " + "; ".join(
errors
)
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": error_message if error_message else "Successfully retrieved tools"
"message": error_message
if error_message
else "Successfully retrieved tools",
}
except Exception as e:
verbose_logger.exception("Unexpected error in list_tool_rest_api: %s", str(e))
verbose_logger.exception(
"Unexpected error in list_tool_rest_api: %s", str(e)
)
return {
"tools": [],
"error": "unexpected_error",
"message": f"An unexpected error occurred: {str(e)}"
"message": f"An unexpected error occurred: {str(e)}",
}
@router.post("/tools/call", dependencies=[Depends(user_api_key_auth)])
@@ -196,9 +222,9 @@ if MCP_AVAILABLE:
detail={
"error": "blocked_pii_entity",
"message": str(e),
"entity_type": getattr(e, 'entity_type', None),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"entity_type": getattr(e, "entity_type", None),
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
@@ -207,8 +233,8 @@ if MCP_AVAILABLE:
detail={
"error": "guardrail_violation",
"message": str(e),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except HTTPException as e:
# Re-raise HTTPException as-is to preserve status code and detail
@@ -220,10 +246,10 @@ if MCP_AVAILABLE:
status_code=500,
detail={
"error": "internal_server_error",
"message": f"An unexpected error occurred: {str(e)}"
}
"message": f"An unexpected error occurred: {str(e)}",
},
)
########################################################
# MCP Connection testing routes
# /health -> Test if we can connect to the MCP server
@@ -234,15 +260,15 @@ if MCP_AVAILABLE:
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
)
async def _execute_with_mcp_client(request: NewMCPServerRequest, operation):
"""
Common helper to create MCP client, execute operation, and ensure proper cleanup.
Args:
request: MCP server configuration
operation: Async function that takes a client and returns the operation result
Returns:
Operation result or error response
"""
@@ -254,15 +280,14 @@ if MCP_AVAILABLE:
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
spec_version=_convert_protocol_version_to_enum(request.spec_version),
auth_type=request.auth_type,
mcp_info=request.mcp_info,
),
mcp_auth_header=None,
)
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
return {"status": "error", "message": "An internal error has occurred."}
@@ -273,6 +298,7 @@ if MCP_AVAILABLE:
await client.disconnect()
except Exception as e:
verbose_logger.warning(f"Error disconnecting MCP client: {e}")
@router.post("/test/connection")
async def test_connection(
request: NewMCPServerRequest,
@@ -280,13 +306,13 @@ if MCP_AVAILABLE:
"""
Test if we can connect to the provided MCP server before adding it
"""
async def _test_connection_operation(client):
await client.connect()
return {"status": "ok"}
return await _execute_with_mcp_client(request, _test_connection_operation)
@router.post("/test/tools/list")
async def test_tools_list(
request: NewMCPServerRequest,
@@ -295,13 +321,16 @@ if MCP_AVAILABLE:
"""
Preview tools available from MCP server before adding it
"""
async def _list_tools_operation(client):
list_tools_result: List[MCPTool] = await client.list_tools()
model_dumped_tools: List[dict] = [tool.model_dump() for tool in list_tools_result]
model_dumped_tools: List[dict] = [
tool.model_dump() for tool in list_tools_result
]
return {
"tools": model_dumped_tools,
"error": None,
"message": "Successfully retrieved tools"
"message": "Successfully retrieved tools",
}
return await _execute_with_mcp_client(request, _list_tools_operation)
litellm/proxy/_experimental/mcp_server/server.py
+143 -77
View File
@@ -130,7 +130,9 @@ if MCP_AVAILABLE:
await _sse_session_manager_cm.__aenter__()
_SESSION_MANAGERS_INITIALIZED = True
verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!")
verbose_logger.info(
"MCP Server started with StreamableHTTP and SSE session managers!"
)
async def shutdown_session_managers():
"""Shutdown the session managers."""
@@ -171,11 +173,18 @@ if MCP_AVAILABLE:
"""
try:
# Get user authentication from context variable
user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version = (
get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_tools - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(f"MCP list_tools - MCP servers from context: {mcp_servers}")
verbose_logger.debug(
f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
@@ -186,9 +195,10 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.info(f"MCP list_tools - Successfully returned {len(tools)} tools")
verbose_logger.info(
f"MCP list_tools - Successfully returned {len(tools)} tools"
)
return tools
except Exception as e:
verbose_logger.exception(f"Error in list_tools endpoint: {str(e)}")
@@ -220,9 +230,16 @@ if MCP_AVAILABLE:
from litellm.proxy.proxy_server import proxy_config
# Validate arguments
user_api_key_auth, mcp_auth_header, _, mcp_server_auth_headers, mcp_protocol_version = get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
_,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(
f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}"
)
try:
# Create a body date for logging
body_data = {"name": name, "arguments": arguments}
@@ -249,17 +266,22 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
**data, # for logging
)
except BlockedPiiEntityError as e:
verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Blocked PII entity detected - {str(e)}", type="text")]
return [
TextContent(
text=f"Error: Blocked PII entity detected - {str(e)}", type="text"
)
]
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")]
return [
TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")
]
except HTTPException as e:
verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
@@ -287,6 +309,7 @@ if MCP_AVAILABLE:
Get the filtered MCP servers from the MCP server names
"""
from typing import Set
filtered_server_ids: Set[str] = set()
# Filter servers based on mcp_servers parameter if provided
if mcp_servers is not None:
@@ -297,7 +320,11 @@ if MCP_AVAILABLE:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
if server:
match_list = [s.lower() for s in [server.alias, server.server_name, server_id] if s is not None]
match_list = [
s.lower()
for s in [server.alias, server.server_name, server_id]
if s is not None
]
if server_or_group.lower() in match_list:
filtered_server_ids.add(server_id)
@@ -306,19 +333,23 @@ if MCP_AVAILABLE:
if not server_name_matched:
try:
access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
access_group_server_ids = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
)
)
# Only include servers that the user has access to
for server_id in access_group_server_ids:
if server_id in allowed_mcp_servers:
filtered_server_ids.add(server_id)
except Exception as e:
verbose_logger.debug(f"Could not resolve '{server_or_group}' as access group: {e}")
verbose_logger.debug(
f"Could not resolve '{server_or_group}' as access group: {e}"
)
if filtered_server_ids:
allowed_mcp_servers = list(filtered_server_ids)
return allowed_mcp_servers
async def _get_tools_from_mcp_servers(
@@ -326,7 +357,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@@ -344,7 +374,9 @@ if MCP_AVAILABLE:
return []
# Get allowed MCP servers based on user permissions
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(
user_api_key_auth
)
if mcp_servers is not None:
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
@@ -352,6 +384,8 @@ if MCP_AVAILABLE:
allowed_mcp_servers=allowed_mcp_servers,
)
# Decide whether to add prefix based on number of allowed servers
add_prefix = not (len(allowed_mcp_servers) == 1)
# Get tools from each allowed server
all_tools = []
@@ -375,15 +409,21 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
add_prefix=add_prefix,
)
all_tools.extend(tools)
verbose_logger.debug(f"Successfully fetched {len(tools)} tools from server {server.name}")
verbose_logger.debug(
f"Successfully fetched {len(tools)} tools from server {server.name}"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from server {server.name}: {str(e)}")
verbose_logger.exception(
f"Error getting tools from server {server.name}: {str(e)}"
)
# Continue with other servers instead of failing completely
verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers")
verbose_logger.info(
f"Successfully fetched {len(all_tools)} tools total from all MCP servers"
)
return all_tools
async def _list_mcp_tools(
@@ -391,7 +431,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all available MCP tools.
@@ -415,11 +454,14 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers")
verbose_logger.debug(
f"Successfully fetched {len(managed_tools)} tools from managed MCP servers"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from managed MCP servers: {str(e)}")
verbose_logger.exception(
f"Error getting tools from managed MCP servers: {str(e)}"
)
# Continue with empty managed tools list instead of failing completely
# Get tools from local registry
@@ -430,10 +472,16 @@ if MCP_AVAILABLE:
# Convert local tools to MCPTool format
for tool in local_tools_raw:
# Convert from litellm.types.mcp_server.tool_registry.MCPTool to mcp.types.Tool
mcp_tool = MCPTool(name=tool.name, description=tool.description, inputSchema=tool.input_schema)
mcp_tool = MCPTool(
name=tool.name,
description=tool.description,
inputSchema=tool.input_schema,
)
local_tools.append(mcp_tool)
except Exception as e:
verbose_logger.exception(f"Error getting tools from local registry: {str(e)}")
verbose_logger.exception(
f"Error getting tools from local registry: {str(e)}"
)
# Continue with empty local tools list instead of failing completely
# Combine all tools
@@ -448,7 +496,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
**kwargs: Any,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""
@@ -456,35 +503,46 @@ if MCP_AVAILABLE:
"""
start_time = datetime.now()
if arguments is None:
raise HTTPException(status_code=400, detail="Request arguments are required")
raise HTTPException(
status_code=400, detail="Request arguments are required"
)
# Remove prefix from tool name for logging and processing
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(name)
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(
name
)
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = _get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = (
_get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
)
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get(
"litellm_logging_obj", None
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj", None)
if litellm_logging_obj:
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model = f"MCP: {name}"
# Try managed server tool first (pass the full prefixed name)
# Primary and recommended way to use MCP servers
#########################################################
mcp_server: Optional[MCPServer] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
mcp_server: Optional[
MCPServer
] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get(
"mcp_server_cost_info"
)
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
response = await _handle_managed_mcp_tool(
name=name, # Pass the full name (potentially prefixed)
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
litellm_logging_obj=litellm_logging_obj,
)
@@ -537,7 +595,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
litellm_logging_obj: Optional[Any] = None,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""Handle tool execution for managed server tools"""
@@ -577,42 +634,47 @@ if MCP_AVAILABLE:
Get the MCP servers from the path
"""
import re
mcp_servers_from_path: Optional[List[str]] = None
# Match /mcp/<servers>/<optional_path>
# Where <servers> can be comma-separated list of server names
# Match /mcp/<servers_and_maybe_path>
# Where servers can be comma-separated list of server names
# Server names can contain slashes (e.g., "custom_solutions/user_123")
mcp_path_match = re.match(r"^/mcp/([^?#]+?)(/[^?#]*)?(?:\?.*)?(?:#.*)?$", path)
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if mcp_path_match:
mcp_servers_str = mcp_path_match.group(1)
optional_path = mcp_path_match.group(2)
if mcp_servers_str:
# First, try to split by comma for comma-separated lists
if ',' in mcp_servers_str:
if "," in mcp_servers_str:
# For comma-separated lists, we need to handle the case where the last item
# might include the path (e.g., "zapier,group1/tools" -> ["zapier", "group1/tools"])
parts = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
# If there's an optional path AND the last part contains a slash that matches the optional path,
# remove the path portion from the last server name
if optional_path and len(parts) > 0 and '/' in parts[-1]:
if optional_path and len(parts) > 0 and "/" in parts[-1]:
last_part = parts[-1]
# Check if the last part ends with the optional path
if optional_path and last_part.endswith(optional_path.lstrip('/')):
if optional_path and last_part.endswith(
optional_path.lstrip("/")
):
# Remove the path portion from the last server name
parts[-1] = last_part[:-len(optional_path.lstrip('/'))]
parts[-1] = last_part[: -len(optional_path.lstrip("/"))]
mcp_servers_from_path = parts
else:
# For single server, it might be just a name or contain slashes
# We need to determine where the server name ends and the path begins
# This is tricky - let's use the original logic but handle comma cases differently
single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", mcp_servers_str)
single_server_match = re.match(
r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", mcp_servers_str
)
if single_server_match:
server_name = single_server_match.group(1)
mcp_servers_from_path = [server_name]
else:
mcp_servers_from_path = [mcp_servers_str]
mcp_servers_from_path = [servers_and_path]
return mcp_servers_from_path
async def extract_mcp_auth_context(scope, path):
@@ -627,7 +689,6 @@ if MCP_AVAILABLE:
mcp_auth_header,
_,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
mcp_servers = mcp_servers_from_path
else:
@@ -636,11 +697,12 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers
async def handle_streamable_http_mcp(scope: Scope, receive: Receive, send: Send) -> None:
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
"""Handle MCP requests through StreamableHTTP."""
try:
path = scope.get("path", "")
@@ -649,20 +711,19 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
# Set the auth context variable for easy access in MCP functions
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
# Ensure session managers are initialized
@@ -686,7 +747,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@@ -699,19 +762,18 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
if not _SESSION_MANAGERS_INITIALIZED:
@@ -733,7 +795,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@@ -769,7 +833,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> None:
"""
Set the UserAPIKeyAuth in the auth context variable.
@@ -785,13 +848,17 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
auth_context_var.set(auth_user)
def get_auth_context() -> Tuple[
Optional[UserAPIKeyAuth], Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]
]:
def get_auth_context() -> (
Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
]
):
"""
Get the UserAPIKeyAuth from the auth context variable.
@@ -806,9 +873,8 @@ if MCP_AVAILABLE:
auth_user.mcp_auth_header,
auth_user.mcp_servers,
auth_user.mcp_server_auth_headers,
auth_user.mcp_protocol_version,
)
return None, None, None, None, None
return None, None, None, None
########################################################
############ End of Auth Context Functions #############
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_buildManifest.js
View File
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_ssgManifest.js
View File
litellm/proxy/_experimental/out/_next/static/chunks/117-a0da667066d322b6.js → litellm/proxy/_experimental/out/_next/static/chunks/117-6a9f6a591033d4a0.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/154-6f752d9e0a5e497b.js → litellm/proxy/_experimental/out/_next/static/chunks/154-b1f2a106d0e0d77b.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/220-8af5927d18414264.js → litellm/proxy/_experimental/out/_next/static/chunks/220-89d73a525e307735.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/50-bb8a11a7610535aa.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/50-d0da2dd7acce2eb9.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-df4939b4f2ec37d7.js
+1
View File
@@ -0,0 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{85210:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_1c856b', '__Inter_Fallback_1c856b'",fontStyle:"normal"},className:"__className_1c856b"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=85210)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-f4acf18888f0aa20.js
-1
View File
@@ -1 +0,0 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-0dbadf20167b786c.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-13b00ef4a072d920.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{67355:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=67355)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-f469bae327299fbb.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);

Some files were not shown because too many files have changed in this diff Show More