tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-23 20:26:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Resolve merge conflicts with main branch

Browse Source 
- Accept main branch's consolidation of Cohere providers
- Preserve header implementation with proper three-tier merging
- Replace outdated header handling in Cohere sections with consolidated approach
- Maintain backward compatibility and functionality
This commit is contained in:
uzaxirr
2025-09-22 05:59:28 +05:30
parent 50d717cde0 be193fbffd
commit d8d5581f29
229 changed files with 15056 additions and 4803 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+77 -4
View File
@@ -1050,6 +1050,51 @@ jobs:
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mapped_tests_coverage.xml
mv .coverage litellm_mapped_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mapped_tests_coverage.xml
- litellm_mapped_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
@@ -1458,6 +1503,7 @@ jobs:
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
@@ -1778,8 +1824,8 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \
-e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e AZURE_API_BASE=$AZURE_API_BASE \
-e AZURE_API_VERSION="2024-05-01-preview" \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@@ -2825,8 +2871,8 @@ jobs:
source "$NVM_DIR/bash_completion"
# Install and use Node version
nvm install v18.17.0
nvm use v18.17.0
nvm install v20
nvm use v20
cd ui/litellm-dashboard
@@ -2879,7 +2925,26 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Run UI unit tests (Vitest)
command: |
# Use Node 20 (several deps require >=20)
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
--coverage.reporter=lcov \
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -3155,6 +3220,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_enterprise_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests:
filters:
branches:
@@ -3199,6 +3270,7 @@ workflows:
- guardrails_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
@@ -3259,6 +3331,7 @@ workflows:
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
Dockerfile
-3
View File
@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
README.md
+1 -1
View File
@@ -37,7 +37,7 @@ LiteLLM manages:
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
+99 -35
View File
@@ -4,10 +4,10 @@ This document provides comprehensive instructions for AI agents to generate rele
## Required Inputs
1. **Release Version** (e.g., `v1.76.3-stable`)
1. **Release Version** (e.g., `v1.77.3-stable`)
2. **PR Diff/Changelog** - List of PRs with titles and contributors
3. **Previous Version Commit Hash** - To compare model pricing changes
4. **Reference Release Notes** - Previous release notes to follow style/format
4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
## Step-by-Step Process
@@ -26,12 +26,12 @@ git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
### 2. Release Notes Structure
Follow this exact structure based on `docs/my-website/release_notes/v1.76.1-stable/index.md`:
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable):
```markdown
---
title: "v1.76.X-stable - [Key Theme]"
slug: "v1-76-X"
title: "v1.77.X-stable - [Key Theme]"
slug: "v1-77-X"
date: YYYY-MM-DDTHH:mm:ss
authors: [standard author block]
hide_table_of_contents: false
@@ -43,23 +43,42 @@ hide_table_of_contents: false
## Key Highlights
[3-5 bullet points of major features]
## Major Changes
[Critical changes users need to know]
## Performance Improvements
[Performance-related changes]
## New Models / Updated Models
[Detailed model tables and provider updates]
#### New Model Support
[Model pricing table]
#### Features
[Provider-specific features organized by provider]
### Bug Fixes
[Provider-specific bug fixes organized by provider]
#### New Provider Support
[New provider integrations]
## LLM API Endpoints
[API-related features and fixes]
#### Features
[API-specific features organized by API type]
#### Bugs
[General bug fixes]
## Management Endpoints / UI
[Admin interface and management changes]
#### Features
[UI and management features]
#### Bugs
[Management-related bug fixes]
## Logging / Guardrail Integrations
[Observability and security features]
#### Features
[Organized by integration provider with proper doc links]
#### Guardrails
[Guardrail-specific features and fixes]
#### New Integration
[Major new integrations]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements]
@@ -86,21 +105,27 @@ hide_table_of_contents: false
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
- Group by provider
- Note pricing corrections
- Highlight deprecated models
- **Structure:**
- `#### New Model Support` - pricing table
- `#### Features` - organized by provider with documentation links
- `### Bug Fixes` - provider-specific bug fixes
- `#### New Provider Support` - major new provider integrations
- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
- Use bullet points under each provider for multiple features
- Separate features from bug fixes clearly
**Provider Features:**
- Group by provider (Gemini, OpenAI, Anthropic, etc.)
- Link to provider docs: `../../docs/providers/[provider_name]`
- Separate features from bug fixes
**API Endpoints:**
- Images API
- Video Generation (if applicable)
- Responses API
- Passthrough endpoints
- General chat completions
**LLM API Endpoints:**
- **Structure:**
- `#### Features` - organized by API type (Responses API, Batch API, etc.)
- `#### Bugs` - general bug fixes under **General** category
- **API Categories:**
- Responses API
- Batch API
- CountTokens API
- Images API
- Video Generation (if applicable)
- General (miscellaneous improvements)
- Use proper documentation links for each API type
**UI/Management:**
- Authentication changes
@@ -108,11 +133,19 @@ hide_table_of_contents: false
- Team management
- Key management
**Integrations:**
- Logging providers (Datadog, Braintrust, etc.)
- Guardrails
- Cost tracking
- Observability
**Logging / Guardrail Integrations:**
- **Structure:**
- `#### Features` - organized by integration provider with proper doc links
- `#### Guardrails` - guardrail-specific features and fixes
- `#### New Integration` - major new integrations
- **Integration Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- Other logging providers with proper doc links
- Use bullet points under each provider for multiple features
- Separate logging features from guardrails clearly
### 4. Documentation Linking Strategy
@@ -211,10 +244,41 @@ This release has a known issue...
:::
```
**Provider Features:**
**Provider Features (New Models / Updated Models section):**
```markdown
#### Features
- **[Provider Name](../../docs/providers/provider)**
- Feature description - [PR #XXXXX](link)
- Another feature description - [PR #YYYYY](link)
```
**API Features (LLM API Endpoints section):**
```markdown
#### Features
- **[API Name](../../docs/api_path)**
- Feature description - [PR #XXXXX](link)
- Another feature - [PR #YYYYY](link)
- **General**
- Miscellaneous improvements - [PR #ZZZZZ](link)
```
**Integration Features (Logging / Guardrail Integrations section):**
```markdown
#### Features
- **[Integration Name](../../docs/proxy/logging#integration)**
- Feature description - [PR #XXXXX](link)
- Bug fix description - [PR #YYYYY](link)
```
**Bug Fixes Pattern:**
```markdown
### Bug Fixes
- **[Provider/Component Name](../../docs/providers/provider)**
- Bug fix description - [PR #XXXXX](link)
```
### 10. Missing Documentation Check
docs/my-website/docs/mcp.md
-2
View File
@@ -114,7 +114,6 @@ mcp_servers:
description: "My custom MCP server"
auth_type: "api_key"
auth_value: "abc123"
spec_version: "2025-03-26"
```
**Configuration Options:**
@@ -716,7 +715,6 @@ mcp_servers:
url: https://mcp.deepwiki.com/mcp
transport: "http"
auth_type: "none"
spec_version: "2025-03-26"
access_groups: ["dev_group"]
```
docs/my-website/docs/observability/helicone_integration.md
+4 -1
View File
@@ -237,7 +237,10 @@ litellm.metadata = {
}
```
### Session Tracking and Tracing
</TabItem>
</Tabs>
## Session Tracking and Tracing
Track multi-step and agentic LLM interactions using session IDs and paths:
docs/my-website/docs/providers/bedrock.md
+53 -1
View File
@@ -1821,6 +1821,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mistral 7B Instruct | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding
### API keys
@@ -1842,11 +1843,29 @@ response = embedding(
print(response)
```
#### Titan V2 - encoding_format support
```python
from litellm import embedding
# Float format (default)
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="float" # Returns float array
)
# Binary format
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="base64" # Returns base64 encoded binary
)
```
## Supported AWS Bedrock Embedding Models
| Model Name | Usage | Supported Additional OpenAI params |
|----------------------|---------------------------------------------|-----|
| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | `dimensions`, `encoding_format` |
| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
@@ -1935,6 +1954,39 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
### Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import image_generation
response = image_generation(
model="bedrock/amazon.nova-canvas-v1:0",
model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
prompt="A cute baby sea otter"
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: nova-canvas-inference-profile
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
aws_region_name: "eu-west-1"
```
</TabItem>
</Tabs>
## Supported AWS Bedrock Image Generation Models
| Model Name | Function Call |
docs/my-website/docs/providers/bedrock_embedding.md
+95
View File
@@ -0,0 +1,95 @@
# Bedrock Embedding
## Supported Embedding Models
| Provider | LiteLLM Route | AWS Documentation |
|----------|---------------|-------------------|
| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) |
| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) |
| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) |
### API keys
This can be set as env variables or passed as **params to litellm.embedding()**
```python
import os
os.environ["AWS_ACCESS_KEY_ID"] = "" # Access key
os.environ["AWS_SECRET_ACCESS_KEY"] = "" # Secret access key
os.environ["AWS_REGION_NAME"] = "" # us-east-1, us-east-2, us-west-1, us-west-2
```
## Usage
### LiteLLM Python SDK
```python
from litellm import embedding
response = embedding(
model="bedrock/amazon.titan-embed-text-v1",
input=["good morning from litellm"],
)
print(response)
```
### LiteLLM Proxy Server
#### 1. Setup config.yaml
```yaml
model_list:
- model_name: titan-embed-v1
litellm_params:
model: bedrock/amazon.titan-embed-text-v1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
- model_name: titan-embed-v2
litellm_params:
model: bedrock/amazon.titan-embed-text-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
#### 2. Start Proxy
```bash
litellm --config /path/to/config.yaml
```
#### 3. Use with OpenAI Python SDK
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.embeddings.create(
input=["good morning from litellm"],
model="titan-embed-v1"
)
print(response)
```
#### 4. Use with LiteLLM Python SDK
```python
import litellm
response = litellm.embedding(
model="titan-embed-v1", # model alias from config.yaml
input=["good morning from litellm"],
api_base="http://0.0.0.0:4000",
api_key="anything"
)
print(response)
```
## Supported AWS Bedrock Embedding Models
| Model Name | Usage | Supported Additional OpenAI params |
|----------------------|---------------------------------------------|-----|
| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
| TwelveLabs Marengo Embed 2.7 | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input)` | Supports multimodal input (text, video, audio, image) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)
### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)
docs/my-website/docs/providers/vertex.md
-144
View File
@@ -2509,150 +2509,6 @@ print("response from proxy", response)
</TabItem>
</Tabs>
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME = "litellm-testing-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-test-project"
```
### Usage
#### 1. Create a file of batch requests for vertex
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**
Each `body` in the file should be an **OpenAI API request**
Create a file called `vertex_batch_completions.jsonl` in the current working directory, the `model` should be the Vertex AI model name
```
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload a File of batch requests
For `vertex_ai` litellm will upload the file to the provided `GCS_BUCKET_NAME`
```python
import os
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_name = "vertex_batch_completions.jsonl" #
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)
file_obj = oai_client.files.create(
file=open(file_path, "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use vertex_ai for this file upload
)
```
**Expected Response**
```json
{
"id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"bytes": 416,
"created_at": 1733392026,
"filename": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"status_details": null
}
```
#### 3. Create a batch
```python
batch_input_file_id = file_obj.id # use `file_obj` from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # example input_file_id = "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/c2b1b785-252b-448c-b180-033c4c63b3ce"
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1733392026,
"endpoint": "",
"input_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
#### 4. Retrieve a batch
```python
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id,
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1736500100,
"endpoint": "",
"input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
## **Fine Tuning APIs**
docs/my-website/docs/providers/vertex_batch.md
+264
View File
@@ -0,0 +1,264 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME="my-batch-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-project"
```
### Usage
Follow this complete workflow: create JSONL file → upload file → create batch → retrieve batch status → get file content
#### 1. Create a JSONL file of batch requests
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**.
Each `body` in the file should be an **OpenAI API request**.
Create a file called `batch_requests.jsonl` with your requests:
```jsonl
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload the file
Upload your JSONL file. For `vertex_ai`, the file will be stored in your configured GCS bucket provided by `GCS_BUCKET_NAME`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="upload_file.py"
from openai import OpenAI
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_obj = oai_client.files.create(
file=open("batch_requests.jsonl", "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"File uploaded with ID: {file_obj.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Upload File"
curl --request POST \
--url http://localhost:4000/v1/files \
--header 'Content-Type: multipart/form-data' \
--form purpose=batch \
--form file=@batch_requests.jsonl \
--form custom_llm_provider=vertex_ai
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"bytes": 416,
"created_at": 1758303684,
"filename": "litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"expires_at": null,
"status_details": null
}
```
#### 3. Create a batch
Create a batch job using the uploaded file ID.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="create_batch.py"
batch_input_file_id = file_obj.id # from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch created with ID: {create_batch_response.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Create Batch Request"
curl --request POST \
--url http://localhost:4000/v1/batches \
--header 'Content-Type: application/json' \
--data '{
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"custom_llm_provider": "vertex_ai"
}'
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite",
"request_counts": null,
"usage": null
}
```
#### 4. Retrieve batch status
Check the status of your batch job. The batch will progress through states: `validating` → `in_progress` → `completed`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="retrieve_batch.py"
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch status: {retrieved_batch.status}")
if retrieved_batch.status == "completed":
print(f"Output file: {retrieved_batch.output_file_id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Retrieve Batch Status"
curl --request GET \
--url 'http://localhost:4000/batches/7814463557919047680?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response (when completed):**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/prediction-model-2025-09-19T21:26:51.569037Z/predictions.jsonl",
"request_counts": null,
"usage": null
}
```
#### 5. Get file content
Once the batch is completed, retrieve the results using the `output_file_id` from the batch response.
**Important:** The `output_file_id` must be URL encoded when used in the request path.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="get_file_content.py"
import urllib.parse
import json
output_file_id = retrieved_batch.output_file_id
# URL encode the file ID
encoded_file_id = urllib.parse.quote_plus(output_file_id)
# Get file content
file_content = oai_client.files.content(
file_id=encoded_file_id,
extra_body={"custom_llm_provider": "vertex_ai"}
)
# Process the results
for line in file_content.text.strip().split('\n'):
result = json.loads(line)
print(f"Request: {result['request']}")
print(f"Response: {result['response']}")
print("---")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Get File Content"
# Note: The file ID must be URL encoded
curl --request GET \
--url 'http://localhost:4000/files/gs%253A%252F%252Fmy-batch-bucket%252Flitellm-vertex-files%252Fpublishers%252Fgoogle%252Fmodels%252Fgemini-2.5-flash-lite%252Fprediction-model-2025-09-19T21%253A26%253A51.569037Z%252Fpredictions.jsonl/content?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response:**
The response contains JSONL format with one result per line:
```jsonl
{"status":"","processed_time":"2025-09-19T21:29:47.352+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are a helpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.48079710006713866,"content":{"parts":[{"text":"Hello there! It's nice to meet you"}],"role":"model"},"finishReason":"MAX_TOKENS"}],"createTime":"2025-09-19T21:29:47.484619Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaIvKHdvshMIP_aOtuAg","usageMetadata":{"candidatesTokenCount":10,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":10}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":19,"trafficType":"ON_DEMAND"}}}
{"status":"","processed_time":"2025-09-19T21:29:47.358+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are an unhelpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.6168075137668185,"content":{"parts":[{"text":"I am unable to assist with this request."}],"role":"model"},"finishReason":"STOP"}],"createTime":"2025-09-19T21:29:47.470889Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaOneHISShMIP28nA8QQ","usageMetadata":{"candidatesTokenCount":9,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":9}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":18,"trafficType":"ON_DEMAND"}}}
```
docs/my-website/docs/proxy/budget_reset_and_tz.md
+1
View File
@@ -29,5 +29,6 @@ Common timezone values:
- `US/Pacific` - Pacific Time
- `Europe/London` - UK Time
- `Asia/Kolkata` - Indian Standard Time (IST)
- `Asia/Bangkok` - Indochina Time (ICT)
- `Asia/Tokyo` - Japan Standard Time
- `Australia/Sydney` - Australian Eastern Time
docs/my-website/docs/proxy/config_settings.md
+3
View File
@@ -690,6 +690,8 @@ router_settings:
| PILLAR_API_KEY | API key for Pillar API Guardrails
| PILLAR_ON_FLAGGED_ACTION | Action to take when content is flagged ('block' or 'monitor')
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
| POSTHOG_API_KEY | API key for PostHog analytics integration
| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -770,3 +772,4 @@ router_settings:
| WEBHOOK_URL | URL for receiving webhooks from external services
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run |
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 |
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 |
docs/my-website/docs/proxy/deploy.md
+1
View File
@@ -13,6 +13,7 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
docs/my-website/docs/proxy/dynamic_rate_limit.md
+241
View File
@@ -0,0 +1,241 @@
# Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
## Quick Start Usage
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python showLineNumbers title="test.py"
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
## [BETA] Set Priority / Reserve Quota
Reserve TPM/RPM capacity for different environments or use cases. This ensures critical production workloads always have guaranteed capacity, while development or lower-priority tasks use remaining quota.
**Use Cases:**
- Production vs Development environments
- Real-time applications vs batch processing
- Critical services vs experimental features
:::tip
Reserving TPM/RPM on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
### How Priority Reservation Works
Priority reservation allocates a percentage of your model's total TPM/RPM to specific priority levels. Keys with higher priority get guaranteed access to their reserved quota first.
**Example Scenario:**
- Model has 10 RPM total capacity
- Priority reservation: `{"prod": 0.9, "dev": 0.1}`
- Result: Production keys get 9 RPM guaranteed, Development keys get 1 RPM guaranteed
### Configuration
#### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 10 # Total model capacity
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
priority_reservation:
"prod": 0.9 # 90% reserved for production (9 RPM)
"dev": 0.1 # 10% reserved for development (1 RPM)
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your.env
```
**Configuration Details:**
`priority_reservation`: Dict[str, float]
- **Key (str)**: Priority level name (can be any string like "prod", "dev", "critical", etc.)
- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
- **Note**: Values should sum to 1.0 or less
**Start Proxy**
```bash
litellm --config /path/to/config.yaml
```
#### 2. Create Keys with Priority Levels
**Production Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "prod"}
}'
```
**Development Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "dev"}
}'
```
**Expected Response for both:**
```json
{
"key": "sk-...",
"metadata": {"priority": "prod"}, // or "dev"
...
}
```
#### 3. Test Priority Allocation
**Test Production Key (should get 9 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-prod-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from prod"}]
}'
```
**Test Development Key (should get 1 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-dev-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from dev"}]
}'
```
### Expected Behavior
With the configuration above:
1. **Production keys** can make up to 9 requests per minute
2. **Development keys** can make up to 1 request per minute
3. Production requests are never blocked by development usage
**Rate Limit Error Example:**
```json
{
"error": {
"message": "Key=sk-dev-... over available RPM=0. Model RPM=10, Reserved RPM for priority 'dev'=1, Active keys=1",
"type": "rate_limit_exceeded",
"code": 429
}
}
```
### Demo Video
This video walks through setting up dynamic rate limiting with priority reservation and locust tests to validate the behavior.
<iframe width="840" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7
" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
docs/my-website/docs/proxy/team_budgets.md
-185
View File
@@ -178,188 +178,3 @@ Expect to see this metric on prometheus to track the Remaining Budget for the te
```shell
litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06
```
### Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
1. Setup config.yaml
```yaml
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
#### ✨ [BETA] Set Priority / Reserve Quota
Reserve tpm/rpm capacity for projects in prod.
:::tip
Reserving tpm/rpm on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 100
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
priority_reservation: {"dev": 0, "prod": 1}
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
priority_reservation:
- Dict[str, float]
- str: can be any string
- float: from 0 to 1. Specify the % of tpm/rpm to reserve for keys of this priority.
**Start Proxy**
```
litellm --config /path/to/config.yaml
```
2. Create a key with that priority
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-D '{
"metadata": {"priority": "dev"} # 👈 KEY CHANGE
}'
```
**Expected Response**
```
{
...
"key": "sk-.."
}
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: sk-...' \ # 👈 key from step 2.
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
**Expected Response**
```
Key=... over available RPM=0. Model RPM=100, Active keys=None
```
docs/my-website/release_notes/v1.76.0-stable/index.md
+1 -1
View File
@@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.76.0-stable - RPS Improvements"
title: "v1.76.0-stable - RPS Improvements"
slug: "v1-76-0"
date: 2025-08-23T10:00:00
authors:
docs/my-website/release_notes/v1.77.2-stable/index.md
+7 -7
View File
@@ -1,5 +1,5 @@
---
title: "[Pre-Release] v1.77.2-stable - Bedrock Batches API"
title: "v1.77.2-stable - Bedrock Batches API"
slug: "v1-77-2"
date: 2025-09-13T10:00:00
authors:
@@ -21,22 +21,22 @@ import TabItem from '@theme/TabItem';
## Deploy this version
:::info
This release is not yet live.
:::
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.2-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.2.post1
```
</TabItem>
docs/my-website/release_notes/v1.77.3-stable/index.md
+258
View File
@@ -0,0 +1,258 @@
---
title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.3.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.3
```
</TabItem>
</Tabs>
---
## Key Highlights
- **+550 RPS Performance Improvements** - Optimizations in request handling and object initialization.
- **Priority Quota Reservation** - Proxy admins can now reserve TPM/RPM capacity for specific keys.
## Priority Quota Reservation
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| SambaNova | `sambanova/deepseek-v3.1` | 128K | $0.90 | $0.90 | Chat completions |
| SambaNova | `sambanova/gpt-oss-120b` | 128K | $0.72 | $0.72 | Chat completions |
| OVHCloud | Various models | Varies | Contact provider | Contact provider | Chat completions |
| CompactifAI | Various models | Varies | Contact provider | Contact provider | Chat completions |
| TwelveLabs | `twelvelabs/marengo-embed-2.7` | 32K | $0.12 | $0.00 | Embeddings |
#### Features
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- New provider support with comprehensive model catalog - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider integration - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
- **[SambaNova](../../docs/providers/sambanova)**
- Added DeepSeek v3.1 and GPT-OSS-120B models - [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
- **[Bedrock](../../docs/providers/bedrock)**
- Cross-region inference profile cost calculation - [PR #14566](https://github.com/BerriAI/litellm/pull/14566)
- AWS external ID parameter support for authentication - [PR #14582](https://github.com/BerriAI/litellm/pull/14582)
- CountTokens API implementation - [PR #14557](https://github.com/BerriAI/litellm/pull/14557)
- Titan V2 encoding_format parameter support - [PR #14687](https://github.com/BerriAI/litellm/pull/14687)
- Nova Canvas image generation inference profiles - [PR #14578](https://github.com/BerriAI/litellm/pull/14578)
- Bedrock Batches API - batch processing support with file upload and request transformation - [PR #14618](https://github.com/BerriAI/litellm/pull/14618)
- Bedrock Twelve Labs embedding provider support - [PR #14697](https://github.com/BerriAI/litellm/pull/14697)
- **[Vertex AI](../../docs/providers/vertex)**
- Gemini labels field provider-aware filtering - [PR #14563](https://github.com/BerriAI/litellm/pull/14563)
- Gemini Batch API support - [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
- **[Volcengine](../../docs/providers/volcengine)**
- Fixed thinking parameters when disabled - [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
- **[Cohere](../../docs/providers/cohere)**
- Handle Generate API deprecation, default to chat endpoints - [PR #14676](https://github.com/BerriAI/litellm/pull/14676)
- **[TwelveLabs](../../docs/providers/twelvelabs)**
- Added Marengo Embed 2.7 embedding support - [PR #14674](https://github.com/BerriAI/litellm/pull/14674)
### Bug Fixes
- **[Bedrock](../../docs/providers/bedrock)**
- Empty arguments handling in tool call invocation - [PR #14583](https://github.com/BerriAI/litellm/pull/14583)
- **[Vertex AI](../../docs/providers/vertex)**
- Avoid deepcopy crash with non-pickleables in Gemini/Vertex - [PR #14418](https://github.com/BerriAI/litellm/pull/14418)
- **[XAI](../../docs/providers/xai)**
- Fix unsupported stop parameter for grok-code models - [PR #14565](https://github.com/BerriAI/litellm/pull/14565)
- **[Gemini](../../docs/providers/gemini)**
- Updated error message for Gemini API - [PR #14589](https://github.com/BerriAI/litellm/pull/14589)
- Fixed 2.5 Flash Image Preview model routing - [PR #14715](https://github.com/BerriAI/litellm/pull/14715)
- API key passing for token counting endpoints - [PR #14744](https://github.com/BerriAI/litellm/pull/14744)
#### New Provider Support
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- Complete provider integration with model catalog and authentication - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider support with documentation - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
---
## LLM API Endpoints
#### Features
- **[/responses](../../docs/response_api)**
- Added cancel endpoint support for non-admin users - [PR #14594](https://github.com/BerriAI/litellm/pull/14594)
- Improved response session handling and cold storage configuration with s3 - [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
- Added OpenAI & Azure /responses/cancel endpoint support - [PR #14561](https://github.com/BerriAI/litellm/pull/14561)
- **General**
- Enhanced rate limit error messages with details - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
- Middle-truncation for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Bugs
- **[/chat/completions](../../docs/completion/input)**
- Fixed completion chat ID handling - [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
- Prevent AttributeError for _get_tags_from_request_kwargs - [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
- **[/responses](../../docs/response_api)**
- Fixed cost calculation - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **General**
- Rate limiter AttributeError fix - [PR #14609](https://github.com/BerriAI/litellm/pull/14609)
---
## Spend Tracking, Budgets and Rate Limiting
- **Responses API Cost Calculation** fix - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **Anthropic Cache Token Pricing** - Separate 1-hour vs 5-minute cache creation costs - [PR #14620](https://github.com/BerriAI/litellm/pull/14620), [PR #14652](https://github.com/BerriAI/litellm/pull/14652)
- **Indochina Time Timezone** support for budget resets - [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
- **Soft Budget Alert Cache Issues** - Resolved soft budget alert cache issues - [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
- **Dynamic Rate Limiter v3** - Priority routing improvements - [PR #14734](https://github.com/BerriAI/litellm/pull/14734)
- **Enhanced Rate Limit Errors** - More detailed error messages - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
---
## Management Endpoints / UI
#### Features
- **Team Member Service Account Keys** - Allow team members to view keys they create - [PR #14619](https://github.com/BerriAI/litellm/pull/14619)
- **Default Budget for JWT Teams** - Auto-assign budgets to generated teams - [PR #14514](https://github.com/BerriAI/litellm/pull/14514)
- **SSO Access Control Groups** - Enhanced token info endpoint integration - [PR #14738](https://github.com/BerriAI/litellm/pull/14738)
- **Health Test Connect Protection** - Restrict access based on model creation permissions - [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Amazon Bedrock Guardrail Info View** - Enhanced logging visualization - [PR #14696](https://github.com/BerriAI/litellm/pull/14696)
#### Bug Fixes
- **SCIM v2** - Fix group PUSH and PUT operations for non-existent members - [PR #14581](https://github.com/BerriAI/litellm/pull/14581)
- **Guardrail View/Edit/Delete** behavior fixes - [PR #14622](https://github.com/BerriAI/litellm/pull/14622)
- **In-Memory Guardrail** update failures - [PR #14653](https://github.com/BerriAI/litellm/pull/14653)
---
## Logging / Guardrail Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Enhanced spend tracking metrics - [PR #14555](https://github.com/BerriAI/litellm/pull/14555)
- Stream support with is_streamed_request parameter - [PR #14673](https://github.com/BerriAI/litellm/pull/14673)
- Fixed tool calls metadata passing - [PR #14531](https://github.com/BerriAI/litellm/pull/14531)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Added logging support for Responses API - [PR #14597](https://github.com/BerriAI/litellm/pull/14597)
- **[Langsmith](../../docs/proxy/logging#langsmith)**
- Langsmith Sampling Rate - Key/Team-level tracing configuration - [PR #14740](https://github.com/BerriAI/litellm/pull/14740)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Multi-worker support improvements - [PR #14530](https://github.com/BerriAI/litellm/pull/14530)
- User email labels in monitoring - [PR #14520](https://github.com/BerriAI/litellm/pull/14520)
- **[Opik](../../docs/proxy/logging#opik)**
- Fixed timezone issue - [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
### Bug Fixes
- **[S3](../../docs/proxy/logging#s3-buckets)**
- Fixed 404 error when using s3_endpoint_url - [PR #14559](https://github.com/BerriAI/litellm/pull/14559)
#### Guardrails
- **Tool Permission Guardrail** - Fine-grained tool access control - [PR #14519](https://github.com/BerriAI/litellm/pull/14519)
- **Bedrock Guardrails** - Selective guarding support with runtime endpoint configuration - [PR #14575](https://github.com/BerriAI/litellm/pull/14575), [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Default Last Message** in guardrails - [PR #14640](https://github.com/BerriAI/litellm/pull/14640)
- **AWS exceptions handling despite 200 response** - [PR #14658](https://github.com/BerriAI/litellm/pull/14658)
#### New Integration
- **[PostHog](../../docs/observability/posthog)** - Complete observability integration for LiteLLM usage tracking and analytics - [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
---
## MCP Gateway
- **MCP Server Alias Parsing** - Multi-part URL path support - [PR #14558](https://github.com/BerriAI/litellm/pull/14558)
- **MCP Filter Recomputation** - After server deletion - [PR #14542](https://github.com/BerriAI/litellm/pull/14542)
- **MCP Gateway Tools List** improvements - [PR #14695](https://github.com/BerriAI/litellm/pull/14695)
---
## Performance / Loadbalancing / Reliability improvements
- **+500 RPS Performance Boost** when sending the `user` field - [PR #14616](https://github.com/BerriAI/litellm/pull/14616)
- **+50 RPS** by removing iscoroutine from hot path - [PR #14649](https://github.com/BerriAI/litellm/pull/14649)
- **7% reduction** in __init__ overhead - [PR #14689](https://github.com/BerriAI/litellm/pull/14689)
- **Generic Object Pool** implementation for better resource management - [PR #14702](https://github.com/BerriAI/litellm/pull/14702)
---
## General Proxy Improvements
- **Middle-Truncation** for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Security
- **Security Update** - Bump aiohttp==3.12.14, fix CVE-2025-53643 - [PR #14638](https://github.com/BerriAI/litellm/pull/14638)
---
## New Contributors
* @luisfucros made their first contribution in [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
* @hanakannzashi made their first contribution in [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
* @eliasto made their first contribution in [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
* @Rasmusafj made their first contribution in [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
* @LingXuanYin made their first contribution in [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
* @ronaldpereira made their first contribution in [PR #14613](https://github.com/BerriAI/litellm/pull/14613)
* @hula-la made their first contribution in [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
* @carlos-marchal-ph made their first contribution in [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
* @akraines made their first contribution in [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
* @mrFranklin made their first contribution in [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
* @tcx4c70 made their first contribution in [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
* @michaeltansg made their first contribution in [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
* @tosi29 made their first contribution in [PR #14725](https://github.com/BerriAI/litellm/pull/14725)
* @gmdfalk made their first contribution in [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
* @FelipeRodriguesGare made their first contribution in [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
* @mritunjaysharma394 made their first contribution in [PR #14678](https://github.com/BerriAI/litellm/pull/14678)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.2.rc.1...v1.77.3.rc.1)**
docs/my-website/sidebars.js
+4 -1
View File
@@ -201,7 +201,7 @@ const sidebars = {
{
type: "category",
label: "Budgets + Rate Limits",
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/customers"],
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/dynamic_rate_limit", "proxy/customers"],
},
{
type: "link",
@@ -392,6 +392,7 @@ const sidebars = {
"providers/vertex",
"providers/vertex_partner",
"providers/vertex_image",
"providers/vertex_batch",
]
},
{
@@ -411,6 +412,7 @@ const sidebars = {
label: "Bedrock",
items: [
"providers/bedrock",
"providers/bedrock_embedding",
"providers/bedrock_agents",
"providers/bedrock_batches",
"providers/bedrock_vector_store",
@@ -522,6 +524,7 @@ const sidebars = {
"completion/batching",
"completion/mock_requests",
"completion/reliable_completions",
"proxy/veo_video_generation",
]
},
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
+6
View File
@@ -109,6 +109,9 @@ class PagerDutyAlerting(SlackAlerting):
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
@@ -191,6 +194,9 @@ class PagerDutyAlerting(SlackAlerting):
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None,
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_user_id=user_api_key_dict.user_id,
enterprise/litellm_enterprise/integrations/prometheus.py
+20 -21
View File
@@ -102,7 +102,9 @@ class PrometheusLogger(CustomLogger):
# "team",
# "team_alias",
# ],
labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
labelnames=self.get_labels_for_metric(
"litellm_llm_api_time_to_first_token_metric"
),
buckets=LATENCY_BUCKETS,
)
@@ -240,14 +242,14 @@ class PrometheusLogger(CustomLogger):
self.litellm_deployment_state = self._gauge_factory(
"litellm_deployment_state",
"LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
labelnames=self.get_labels_for_metric("litellm_deployment_state")
labelnames=self.get_labels_for_metric("litellm_deployment_state"),
)
self.litellm_deployment_cooled_down = self._counter_factory(
"litellm_deployment_cooled_down",
"LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
# labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down"),
)
self.litellm_deployment_success_responses = self._counter_factory(
@@ -1039,20 +1041,12 @@ class PrometheusLogger(CustomLogger):
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
metric_name="litellm_spend_metric"
),
enum_values=enum_values,
)
self.litellm_spend_metric.labels(
end_user_id,
user_api_key,
user_api_key_alias,
model,
user_api_team,
user_api_team_alias,
user_id,
).inc(response_cost)
self.litellm_spend_metric.labels(**_labels).inc(response_cost)
def _set_virtual_key_rate_limit_metrics(
self,
@@ -2280,7 +2274,9 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
return result
def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: str) -> bool:
def _tag_matches_wildcard_configured_pattern(
tags: List[str], configured_tag: str
) -> bool:
"""
Check if any of the request tags matches a wildcard configured pattern
@@ -2305,6 +2301,7 @@ def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: st
import re
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
pattern_router = PatternMatchRouter()
regex_pattern = pattern_router._pattern_to_regex(configured_tag)
return any(re.match(pattern=regex_pattern, string=tag) for tag in tags)
@@ -2313,11 +2310,11 @@ def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: st
def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
"""
Get custom labels from tags based on admin configuration.
Supports both exact matches and wildcard patterns:
- Exact match: "prod" matches "prod" exactly
- Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"
- Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"
Reuses PatternMatchRouter for wildcard pattern matching.
Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise
@@ -2345,17 +2342,19 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
for configured_tag in configured_tags:
label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}")
# Check for exact match first (backwards compatibility)
if configured_tag in tags:
result[label_name] = "true"
continue
# Use PatternMatchRouter for wildcard pattern matching
if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag):
if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(
tags=tags, configured_tag=configured_tag
):
result[label_name] = "true"
continue
# No match found
result[label_name] = "false"
enterprise/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.19"
version = "0.1.20"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.19"
version = "0.1.20"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20250918083359_drop_spec_version_column_from_mcp_table/migration.sql
+8
View File
@@ -0,0 +1,8 @@
/*
Warnings:
- You are about to drop the column `spec_version` on the `LiteLLM_MCPServerTable` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "public"."LiteLLM_MCPServerTable" DROP COLUMN "spec_version";
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
-1
View File
@@ -171,7 +171,6 @@ model LiteLLM_MCPServerTable {
description String?
url String?
transport String @default("sse")
spec_version String @default("2025-03-26")
auth_type String?
created_at DateTime? @default(now()) @map("created_at")
created_by String?
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.18"
version = "0.2.19"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.18"
version = "0.2.19"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+2 -1
View File
@@ -67,6 +67,7 @@ from litellm.constants import (
bedrock_embedding_models,
known_tokenizer_config,
BEDROCK_INVOKE_PROVIDERS_LITERAL,
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
BEDROCK_CONVERSE_MODELS,
DEFAULT_MAX_TOKENS,
DEFAULT_SOFT_BUDGET,
@@ -116,6 +117,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"logfire",
"literalai",
"dynamic_rate_limiter",
"dynamic_rate_limiter_v3",
"langsmith",
"prometheus",
"otel",
@@ -1045,7 +1047,6 @@ from .llms.databricks.chat.transformation import DatabricksConfig
from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig
from .llms.predibase.chat.transformation import PredibaseConfig
from .llms.replicate.chat.transformation import ReplicateConfig
from .llms.cohere.completion.transformation import CohereTextConfig as CohereConfig
from .llms.snowflake.chat.transformation import SnowflakeConfig
from .llms.cohere.rerank.transformation import CohereRerankConfig
from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config
litellm/caching/redis_cache.py
+2 -1
View File
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.services import ServiceTypes
@@ -138,7 +139,7 @@ class RedisCache(BaseCache):
self.redis_flush_size = redis_flush_size
self.redis_version = "Unknown"
try:
if not inspect.iscoroutinefunction(self.redis_client):
if not coroutine_checker.is_async_callable(self.redis_client):
self.redis_version = self.redis_client.info()["redis_version"] # type: ignore
except Exception:
pass
litellm/constants.py
+13 -1
View File
@@ -179,7 +179,7 @@ NON_LLM_CONNECTION_TIMEOUT = int(
os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15)
) # timeout for adjacent services (e.g. jwt auth)
MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000))
MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 1000))
MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048))
BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
REPLICATE_POLLING_DELAY_SECONDS = float(
os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
@@ -769,6 +769,12 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"deepseek_r1",
]
BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
"cohere",
"amazon",
"twelvelabs",
]
BEDROCK_CONVERSE_MODELS = [
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
@@ -822,6 +828,7 @@ bedrock_embedding_models: set = set(
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
"twelvelabs.marengo-embed-2-7-v1:0",
]
)
@@ -1063,3 +1070,8 @@ SENTRY_PII_DENYLIST = [
"SMTP_SENDER_EMAIL",
"TEST_EMAIL_ADDRESS",
]
# CoroutineChecker cache configuration
COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)
)
litellm/experimental_mcp_client/client.py
+28 -31
View File
@@ -19,8 +19,6 @@ from litellm._logging import verbose_logger
from litellm.types.mcp import (
MCPAuth,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPStdioConfig,
MCPTransport,
MCPTransportType,
@@ -48,7 +46,6 @@ class MCPClient:
auth_value: Optional[str] = None,
timeout: float = 60.0,
stdio_config: Optional[MCPStdioConfig] = None,
protocol_version: MCPSpecVersionType = MCPSpecVersion.jun_2025,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@@ -62,7 +59,6 @@ class MCPClient:
self._session_ctx = None
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.protocol_version: MCPSpecVersionType = protocol_version
# handle the basic auth value if provided
if auth_value:
@@ -84,22 +80,24 @@ class MCPClient:
"""Initialize the transport and session."""
if self._session:
return # Already connected
try:
if self.transport_type == MCPTransport.stdio:
# For stdio transport, use stdio_client with command-line parameters
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {})
env=self.stdio_config.get("env", {}),
)
self._transport_ctx = stdio_client(server_params)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
elif self.transport_type == MCPTransport.sse:
@@ -110,7 +108,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
else: # http
@@ -121,7 +121,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
except ValueError as e:
@@ -184,8 +186,10 @@ class MCPClient:
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers = {}
headers = {
"MCP-Protocol-Version": "2025-06-18"
}
if self._mcp_auth_value:
if self.auth_type == MCPAuth.bearer_token:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
@@ -196,18 +200,8 @@ class MCPClient:
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
# Handle protocol version - it might be a string or enum
if hasattr(self.protocol_version, 'value'):
# It's an enum
protocol_version_str = self.protocol_version.value
else:
# It's a string
protocol_version_str = str(self.protocol_version)
headers["MCP-Protocol-Version"] = protocol_version_str
return headers
async def list_tools(self) -> List[MCPTool]:
"""List available tools from the server."""
if not self._session:
@@ -216,7 +210,7 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return []
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return []
@@ -245,17 +239,20 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")],
isError=True
content=[TextContent(type="text", text=f"{str(e)}")], isError=True
)
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return MCPCallToolResult(
content=[TextContent(type="text", text="MCP client session is not initialized")],
content=[
TextContent(
type="text", text="MCP client session is not initialized"
)
],
isError=True,
)
try:
tool_result = await self._session.call_tool(
name=call_tool_request_params.name,
@@ -270,8 +267,8 @@ class MCPClient:
await self.disconnect()
# Return a default error result instead of raising
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")], # Empty content for error case
content=[
TextContent(type="text", text=f"{str(e)}")
], # Empty content for error case
isError=True,
)
litellm/files/main.py
+27 -1
View File
@@ -731,7 +731,7 @@ def file_list(
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@@ -887,6 +887,32 @@ def file_content(
client=client,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base = optional_params.api_base or ""
vertex_ai_project = (
optional_params.vertex_project
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.vertex_location
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
response = vertex_ai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
timeout=timeout,
max_retries=optional_params.max_retries,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai'.".format(
litellm/integrations/custom_guardrail.py
+4 -1
View File
@@ -357,6 +357,7 @@ class CustomGuardrail(CustomLogger):
end_time: Optional[float] = None,
duration: Optional[float] = None,
masked_entity_count: Optional[Dict[str, int]] = None,
guardrail_provider: Optional[str] = None,
) -> None:
"""
Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
@@ -367,6 +368,7 @@ class CustomGuardrail(CustomLogger):
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,
guardrail_mode=(
GuardrailMode(**self.event_hook.model_dump()) # type: ignore
if isinstance(self.event_hook, Mode)
@@ -487,7 +489,8 @@ class CustomGuardrail(CustomLogger):
"""
Update the guardrails litellm params in memory
"""
pass
for key, value in vars(litellm_params).items():
setattr(self, key, value)
def log_guardrail_information(func):
litellm/integrations/datadog/datadog_llm_obs.py
+26
View File
@@ -498,6 +498,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"guardrail_information": standard_logging_payload.get(
"guardrail_information", None
),
"is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload),
}
#########################################################
@@ -561,6 +562,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
return latency_metrics
def _get_stream_value_from_payload(self, standard_logging_payload: StandardLoggingPayload) -> bool:
"""
Extract the stream value from standard logging payload.
The stream field in StandardLoggingPayload is only set to True for completed streaming responses.
For non-streaming requests, it's None. The original stream parameter is in model_parameters.
Returns:
bool: True if this was a streaming request, False otherwise
"""
# Check top-level stream field first (only True for completed streaming)
stream_value = standard_logging_payload.get("stream")
if stream_value is True:
return True
# Fallback to model_parameters.stream for original request parameters
model_params = standard_logging_payload.get("model_parameters", {})
if isinstance(model_params, dict):
stream_value = model_params.get("stream")
if stream_value is True:
return True
# Default to False for non-streaming requests
return False
def _get_spend_metrics(
self, standard_logging_payload: StandardLoggingPayload
) -> DDLLMObsSpendMetrics:
litellm/integrations/langsmith.py
+34 -24
View File
@@ -39,6 +39,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_api_key: Optional[str] = None,
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_sampling_rate: Optional[float] = None,
**kwargs,
):
self.flush_lock = asyncio.Lock()
@@ -49,7 +50,8 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url=langsmith_base_url,
)
self.sampling_rate: float = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
langsmith_sampling_rate
or float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
@@ -76,26 +78,14 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url: Optional[str] = None,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
if _credentials_api_key is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_api_key=None."
)
_credentials_project = (
langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
)
if _credentials_project is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_project=None."
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
if _credentials_base_url is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_base_url=None."
)
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
@@ -200,12 +190,7 @@ class LangsmithLogger(CustomBatchLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
)
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -219,6 +204,7 @@ class LangsmithLogger(CustomBatchLogger):
kwargs,
response_obj,
)
credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
data = self._prepare_log_data(
kwargs=kwargs,
@@ -245,7 +231,7 @@ class LangsmithLogger(CustomBatchLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -286,7 +272,7 @@ class LangsmithLogger(CustomBatchLogger):
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@@ -417,6 +403,17 @@ class LangsmithLogger(CustomBatchLogger):
for queue_object in self.log_queue:
credentials = queue_object["credentials"]
# if credential missing, skip - log warning
if (
credentials["LANGSMITH_API_KEY"] is None
or credentials["LANGSMITH_PROJECT"] is None
):
verbose_logger.warning(
"Langsmith Logging - credentials missing - api_key: %s, project: %s",
credentials["LANGSMITH_API_KEY"],
credentials["LANGSMITH_PROJECT"],
)
continue
key = CredentialsKey(
api_key=credentials["LANGSMITH_API_KEY"],
project=credentials["LANGSMITH_PROJECT"],
@@ -432,6 +429,19 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
sampling_rate: float = self.sampling_rate
if standard_callback_dynamic_params is not None:
_sampling_rate = standard_callback_dynamic_params.get(
"langsmith_sampling_rate"
)
if _sampling_rate is not None:
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(
self, kwargs: Dict[str, Any]
) -> LangsmithCredentialsObject:
@@ -442,9 +452,9 @@ class LangsmithLogger(CustomBatchLogger):
Otherwise, use the default credentials.
"""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
credentials = self.get_credentials_from_env(
langsmith_api_key=standard_callback_dynamic_params.get(
litellm/integrations/opik/opik.py
+5 -4
View File
@@ -3,6 +3,7 @@ Opik Logger that logs LLM events to an Opik server
"""
import asyncio
from datetime import timezone
import json
import traceback
from typing import Dict, List
@@ -291,8 +292,8 @@ class OpikLogger(CustomBatchLogger):
"project_name": project_name,
"id": trace_id,
"name": trace_name,
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,
@@ -312,8 +313,8 @@ class OpikLogger(CustomBatchLogger):
"parent_span_id": parent_span_id,
"name": span_name,
"type": "llm",
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,
litellm/litellm_core_utils/cached_imports.py
+56
View File
@@ -0,0 +1,56 @@
"""
Cached imports module for LiteLLM.
This module provides cached import functionality to avoid repeated imports
inside functions that are critical to performance.
"""
from typing import TYPE_CHECKING, Callable, Optional, Type
# Type annotations for cached imports
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.coroutine_checker import CoroutineChecker
# Global cache variables
_LiteLLMLogging: Optional[Type["Logging"]] = None
_coroutine_checker: Optional["CoroutineChecker"] = None
_set_callbacks: Optional[Callable] = None
def get_litellm_logging_class() -> Type["Logging"]:
"""Get the cached LiteLLM Logging class, initializing if needed."""
global _LiteLLMLogging
if _LiteLLMLogging is not None:
return _LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import Logging
_LiteLLMLogging = Logging
return _LiteLLMLogging
def get_coroutine_checker() -> "CoroutineChecker":
"""Get the cached coroutine checker instance, initializing if needed."""
global _coroutine_checker
if _coroutine_checker is not None:
return _coroutine_checker
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
_coroutine_checker = coroutine_checker
return _coroutine_checker
def get_set_callbacks() -> Callable:
"""Get the cached set_callbacks function, initializing if needed."""
global _set_callbacks
if _set_callbacks is not None:
return _set_callbacks
from litellm.litellm_core_utils.litellm_logging import set_callbacks
_set_callbacks = set_callbacks
return _set_callbacks
def clear_cached_imports() -> None:
"""Clear all cached imports. Useful for testing or memory management."""
global _LiteLLMLogging, _coroutine_checker, _set_callbacks
_LiteLLMLogging = None
_coroutine_checker = None
_set_callbacks = None
litellm/litellm_core_utils/coroutine_checker.py
+63
View File
@@ -0,0 +1,63 @@
# CoroutineChecker utility for checking if functions/callables are coroutines or coroutine functions
import inspect
from typing import Any
from weakref import WeakKeyDictionary
from litellm.constants import (
COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY,
)
class CoroutineChecker:
"""Utility class for checking coroutine status of functions and callables.
Simple bounded cache using WeakKeyDictionary to avoid memory leaks.
"""
def __init__(self):
self._cache = WeakKeyDictionary()
self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY
def is_async_callable(self, callback: Any) -> bool:
"""Fast, cached check for whether a callback is an async function.
Falls back gracefully if the object cannot be weak-referenced or cached.
2.59x speedup.
"""
# Fast path: check cache first (most common case)
try:
cached = self._cache.get(callback)
if cached is not None:
return cached
except Exception:
pass
# Determine target - optimized path for common cases
target = callback
if not inspect.isfunction(target) and not inspect.ismethod(target):
try:
call_attr = getattr(target, "__call__", None)
if call_attr is not None:
target = call_attr
except Exception:
pass
# Compute result
try:
result = inspect.iscoroutinefunction(target)
except Exception:
result = False
# Cache the result with size enforcement
try:
# Simple size enforcement: clear cache if it gets too large
if len(self._cache) >= self._max_size:
self._cache.clear()
self._cache[callback] = result
except Exception:
pass
return result
# Global instance for backward compatibility and convenience
coroutine_checker = CoroutineChecker()
litellm/litellm_core_utils/custom_logger_registry.py
+2
View File
@@ -47,6 +47,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
VectorStorePreCallHook,
)
from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
class CustomLoggerRegistry:
@@ -86,6 +87,7 @@ class CustomLoggerRegistry:
"s3_v2": S3Logger,
"aws_sqs": SQSLogger,
"dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
"dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
"vector_store_pre_call_hook": VectorStorePreCallHook,
"dotprompt": DotpromptManager,
"cloudzero": CloudZeroLogger,
litellm/litellm_core_utils/duration_parser.py
+1
View File
@@ -158,6 +158,7 @@ def _setup_timezone(
"US/Eastern": timezone(timedelta(hours=-4)), # EDT
"US/Pacific": timezone(timedelta(hours=-7)), # PDT
"Asia/Kolkata": timezone(timedelta(hours=5, minutes=30)), # IST
"Asia/Bangkok": timezone(timedelta(hours=7)), # ICT (Indochina Time)
"Europe/London": timezone(timedelta(hours=1)), # BST
"UTC": timezone.utc,
}
litellm/litellm_core_utils/exception_mapping_utils.py
+71 -99
View File
@@ -6,6 +6,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import LlmProviders
from ..exceptions import (
APIConnectionError,
@@ -556,7 +557,7 @@ def exception_type( # type: ignore # noqa: PLR0915
model=model,
llm_provider="anthropic",
)
elif "overloaded_error" in error_str:
elif "overloaded_error" in error_str or "Overloaded" in error_str:
exception_mapping_worked = True
raise InternalServerError(
message="AnthropicError - {}".format(error_str),
@@ -762,7 +763,7 @@ def exception_type( # type: ignore # noqa: PLR0915
error_str += "XXXXXXX" + '"'
raise AuthenticationError(
message=f"{custom_llm_provider}Exception: Authentication Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@@ -771,14 +772,14 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "model's maximum context limit" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"{custom_llm_provider}Exception: Context Window Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
elif "token_quota_reached" in error_str:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception: Rate Limit Errror - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@@ -789,14 +790,14 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif "model_no_support_for_function" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -804,7 +805,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -814,28 +815,28 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -846,7 +847,7 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -854,7 +855,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -862,7 +863,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 503:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -870,7 +871,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 504: # gateway timeout error
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -1168,9 +1169,9 @@ def exception_type( # type: ignore # noqa: PLR0915
exception_status_code=original_exception.status_code,
)
elif (
custom_llm_provider == "vertex_ai"
or custom_llm_provider == "vertex_ai_beta"
or custom_llm_provider == "gemini"
custom_llm_provider == LlmProviders.VERTEX_AI
or custom_llm_provider == LlmProviders.VERTEX_AI_BETA
or custom_llm_provider == LlmProviders.GEMINI
):
if (
"Vertex AI API has not been used in project" in error_str
@@ -1178,9 +1179,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"litellm.BadRequestError: VertexAIException - {error_str}",
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
request=httpx.Request(
@@ -1193,7 +1194,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"VertexException - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
@@ -1203,9 +1204,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=500,
content=str(original_exception),
@@ -1216,7 +1217,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "API key not valid." in error_str:
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@@ -1224,9 +1225,9 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "403" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=403,
request=httpx.Request(
@@ -1243,9 +1244,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise ContentPolicyViolationError(
message=f"VertexAIException ContentPolicyViolationError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@@ -1264,9 +1265,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@@ -1282,18 +1283,18 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@@ -1306,21 +1307,35 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 401:
exception_mapping_worked = True
raise AuthenticationError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 403:
exception_mapping_worked = True
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=httpx.Response(
status_code=403,
request=httpx.Request(
method="POST",
url="https://cloud.google.com/vertex-ai/",
),
),
)
if original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@@ -1328,9 +1343,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@@ -1343,9 +1358,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"VertexAIException InternalServerError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=500,
@@ -1353,71 +1368,20 @@ def exception_type( # type: ignore # noqa: PLR0915
request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
if original_exception.status_code == 503:
if original_exception.status_code == 502:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"VertexAIException - {original_exception.message}",
raise APIConnectionError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
if "503 Getting metadata" in error_str:
# auth errors look like this
# 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate.
exception_mapping_worked = True
raise BadRequestError(
message="GeminiException - Invalid api key",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"504 Deadline expired before operation could complete." in error_str
or "504 Deadline Exceeded" in error_str
):
exception_mapping_worked = True
raise Timeout(
message=f"GeminiException - {original_exception.message}",
model=model,
llm_provider="palm",
exception_status_code=original_exception.status_code,
)
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"GeminiException - {error_str}",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"500 An internal error has occurred." in error_str
or "list index out of range" in error_str
):
exception_mapping_worked = True
raise APIError(
status_code=getattr(original_exception, "status_code", 500),
message=f"GeminiException - {original_exception.message}",
llm_provider="palm",
model=model,
request=httpx.Response(
status_code=429,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
),
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
if original_exception.status_code == 503:
exception_mapping_worked = True
raise BadRequestError(
message=f"GeminiException - {error_str}",
raise ServiceUnavailableError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
# Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes
elif custom_llm_provider == "cloudflare":
if "Authentication error" in error_str:
exception_mapping_worked = True
@@ -1449,6 +1413,14 @@ def exception_type( # type: ignore # noqa: PLR0915
model=model,
response=getattr(original_exception, "response", None),
)
elif "invalid type: parameter" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"CohereException - {original_exception.message}",
llm_provider="cohere",
model=model,
response=getattr(original_exception, "response", None),
)
elif "too many tokens" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
litellm/litellm_core_utils/get_supported_openai_params.py
+1 -3
View File
@@ -94,9 +94,7 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.VLLMConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "deepseek":
return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "cohere":
return litellm.CohereConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "cohere_chat":
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
return litellm.CohereChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "maritalk":
return litellm.MaritalkConfig().get_supported_openai_params(model=model)
litellm/litellm_core_utils/litellm_logging.py
+41 -10
View File
@@ -3444,6 +3444,30 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj)
return dynamic_rate_limiter_obj # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
if internal_usage_cache is None:
raise Exception(
"Internal Error: Cache cannot be empty - internal_usage_cache={}".format(
internal_usage_cache
)
)
dynamic_rate_limiter_obj_v3 = _PROXY_DynamicRateLimitHandlerV3(
internal_usage_cache=internal_usage_cache
)
if llm_router is not None and isinstance(llm_router, litellm.Router):
dynamic_rate_limiter_obj_v3.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj_v3)
return dynamic_rate_limiter_obj_v3 # type: ignore
elif logging_integration == "langtrace":
if "LANGTRACE_API_KEY" not in os.environ:
raise ValueError("LANGTRACE_API_KEY not found in environment variables")
@@ -3707,6 +3731,14 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandler):
return callback # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
elif logging_integration == "langtrace":
from litellm.integrations.opentelemetry import OpenTelemetry
@@ -3905,22 +3937,25 @@ class StandardLoggingPayloadSetup:
clean_metadata = StandardLoggingMetadata(
user_api_key_hash=None,
user_api_key_alias=None,
user_api_key_spend=None,
user_api_key_max_budget=None,
user_api_key_budget_reset_at=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
user_api_key_user_email=None,
user_api_key_end_user_id=None,
user_api_key_request_route=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
prompt_management_metadata=prompt_management_metadata,
applied_guardrails=applied_guardrails,
mcp_tool_call_metadata=mcp_tool_call_metadata,
vector_store_request_metadata=vector_store_request_metadata,
usage_object=usage_object,
requester_custom_headers=None,
user_api_key_request_route=None,
cold_storage_object_key=None,
)
if isinstance(metadata, dict):
@@ -4583,14 +4618,10 @@ def get_standard_logging_metadata(
cold_storage_object_key=None,
)
if isinstance(metadata, dict):
# Filter the metadata dictionary to include only the specified keys
clean_metadata = StandardLoggingMetadata(
**{ # type: ignore
key: metadata[key]
for key in StandardLoggingMetadata.__annotations__.keys()
if key in metadata
}
)
# Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
for key in StandardLoggingMetadata.__annotations__.keys():
if key in metadata:
clean_metadata[key] = metadata[key] # type: ignore
if metadata.get("user_api_key") is not None:
if is_valid_sha256_hash(str(metadata.get("user_api_key"))):
litellm/litellm_core_utils/llm_cost_calc/utils.py
+301 -123
View File
@@ -1,11 +1,12 @@
# What is this?
## Helper utilities for cost_per_token()
from typing import Any, Literal, Optional, Tuple, cast
from typing import Any, Literal, Optional, Tuple, TypedDict, cast
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
ImageResponse,
ModelInfo,
@@ -113,20 +114,34 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
def _get_token_base_cost(
model_info: ModelInfo, usage: Usage
) -> Tuple[float, float, float, float, float]:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
then we use the corresponding threshold cost for all token types.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, "input_cost_per_token")
)
completion_base_cost = cast(
float, _get_cost_per_unit(model_info, "output_cost_per_token")
)
cache_creation_cost = cast(
float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")
)
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
)
cache_read_cost = cast(
float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")
)
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@@ -140,34 +155,57 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
)
if usage.prompt_tokens > threshold:
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, key, prompt_base_cost))
completion_base_cost = cast(float, _get_cost_per_unit(
model_info,
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
))
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, key, prompt_base_cost)
)
completion_base_cost = cast(
float,
_get_cost_per_unit(
model_info,
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
),
)
# Apply tiered pricing to cache costs
cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
cache_creation_tiered_key = (
f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
)
cache_read_tiered_key = (
f"cache_read_input_token_cost_above_{threshold_str}_tokens"
)
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(float, _get_cost_per_unit(
model_info, cache_creation_tiered_key, cache_creation_cost
))
cache_creation_cost = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_tiered_key,
cache_creation_cost,
),
)
if cache_read_tiered_key in model_info:
cache_read_cost = cast(float, _get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
))
cache_read_cost = cast(
float,
_get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
),
)
break
except (IndexError, ValueError):
continue
except Exception:
continue
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
return (
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
)
def calculate_cost_component(
@@ -195,7 +233,9 @@ def calculate_cost_component(
return 0.0
def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0) -> Optional[float]:
def _get_cost_per_unit(
model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0
) -> Optional[float]:
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
cost_per_unit = model_info.get(cost_key)
if isinstance(cost_per_unit, float):
@@ -210,7 +250,196 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Opti
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
)
return default_value
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: Optional[CacheCreationTokenDetails],
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m = (
cache_creation_token_details.ephemeral_5m_input_tokens
)
cache_creation_tokens_1h = (
cache_creation_token_details.ephemeral_1h_input_tokens
)
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += (
cache_creation_tokens_5m * cache_creation_cost
if cache_creation_tokens_5m is not None
else 0.0
)
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr
if cache_creation_tokens_1h is not None
else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_creation_tokens: int
cache_creation_token_details: Optional[CacheCreationTokenDetails]
text_tokens: int
audio_tokens: int
character_count: int
image_count: int
video_length_seconds: int
def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cache_hit_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
or 0
)
cache_creation_tokens = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
)
or 0
)
cache_creation_token_details = (
cast(
Optional[CacheCreationTokenDetails],
getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
)
or None
)
text_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
character_count=character_count,
image_count=image_count,
video_length_seconds=video_length_seconds,
)
class CompletionTokensDetailsResult(TypedDict):
audio_tokens: int
text_tokens: int
reasoning_tokens: int
def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
return CompletionTokensDetailsResult(
audio_tokens=audio_tokens,
text_tokens=text_tokens,
reasoning_tokens=reasoning_tokens,
)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
"""
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details[
"cache_creation_token_details"
],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
)
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", prompt_tokens_details["character_count"]
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_video_per_second",
prompt_tokens_details["video_length_seconds"],
)
return prompt_cost
def generic_cost_per_token(
@@ -236,83 +465,45 @@ def generic_cost_per_token(
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
prompt_cost = 0.0
### PROCESSING COST
text_tokens = usage.prompt_tokens
cache_hit_tokens = 0
audio_tokens = 0
character_count = 0
image_count = 0
video_length_seconds = 0
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0,
)
if usage.prompt_tokens_details:
cache_hit_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0)
)
or 0
)
text_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None)
)
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0))
or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
prompt_tokens_details = _parse_prompt_tokens_details(usage)
## EDGE CASE - text tokens not set inside PromptTokensDetails
if text_tokens == 0:
text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
model_info=model_info, usage=usage
)
if prompt_tokens_details["text_tokens"] == 0:
text_tokens = (
usage.prompt_tokens
- prompt_tokens_details["cache_hit_tokens"]
- prompt_tokens_details["audio_tokens"]
- prompt_tokens_details["cache_creation_tokens"]
)
prompt_tokens_details["text_tokens"] = text_tokens
prompt_cost = float(text_tokens) * prompt_base_cost
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(model_info=model_info, usage=usage)
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", character_count
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", image_count
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_video_per_second", video_length_seconds
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
prompt_base_cost=prompt_base_cost,
cache_read_cost=cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
)
## CALCULATE OUTPUT COST
@@ -321,27 +512,10 @@ def generic_cost_per_token(
reasoning_tokens = 0
is_text_tokens_total = False
if usage.completion_tokens_details is not None:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
completion_tokens_details = _parse_completion_tokens_details(usage)
audio_tokens = completion_tokens_details["audio_tokens"]
text_tokens = completion_tokens_details["text_tokens"]
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
if text_tokens == 0:
text_tokens = usage.completion_tokens
@@ -350,8 +524,12 @@ def generic_cost_per_token(
## TEXT COST
completion_cost = float(text_tokens) * completion_base_cost
_output_cost_per_audio_token = _get_cost_per_unit(model_info, "output_cost_per_audio_token", None)
_output_cost_per_reasoning_token = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
_output_cost_per_audio_token = _get_cost_per_unit(
model_info, "output_cost_per_audio_token", None
)
_output_cost_per_reasoning_token = _get_cost_per_unit(
model_info, "output_cost_per_reasoning_token", None
)
## AUDIO COST
if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
@@ -397,7 +575,7 @@ class CostCalculatorUtils:
]:
return True
return False
@staticmethod
def route_image_generation_cost_calculator(
model: str,
litellm/litellm_core_utils/object_pooling.py
+137
View File
@@ -0,0 +1,137 @@
"""
Generic object pooling utilities for LiteLLM.
This module provides a flexible object pooling system that can be used
to pool any type of object, reducing memory allocation overhead and
improving performance for frequently created/destroyed objects.
Memory Management Strategy:
- Balanced eviction-based memory control to optimize reuse ratio
- Moderate eviction frequency (300s) to maintain high object reuse
- Conservative eviction weight (0.3) to avoid destroying useful objects
- Lower pre-warm count (5) to reduce initial memory footprint
- Always keeps at least one object available for high availability
- Unlimited pools when maxsize is not specified (eviction controls actual usage)
"""
from typing import Any, Callable, Optional, Type, TypeVar
from pond import Pond, PooledObject, PooledObjectFactory
T = TypeVar('T')
class GenericPooledObjectFactory(PooledObjectFactory):
"""Generic factory class for creating pooled objects of any type."""
def __init__(
self,
object_class: Type[T],
pooled_maxsize: Optional[int] = None, # None = unlimited pool with eviction-based memory control
least_one: bool = True, # Always keep at least one for high concurrency
initializer: Optional[Callable[[T], None]] = None
):
# Only pass maxsize to Pond if user specified it - otherwise let Pond handle unlimited pools
if pooled_maxsize is not None:
super().__init__(pooled_maxsize=pooled_maxsize, least_one=least_one)
else:
super().__init__(least_one=least_one)
self.object_class = object_class
self.initializer = initializer
self._user_maxsize = pooled_maxsize # Store original user preference
def createInstance(self) -> PooledObject:
"""Create a new instance wrapped in a PooledObject."""
# Create a properly initialized instance
obj = self.object_class()
return PooledObject(obj)
def destroy(self, pooled_object: PooledObject):
"""Destroy the pooled object."""
if hasattr(pooled_object.keeped_object, '__dict__'):
pooled_object.keeped_object.__dict__.clear()
del pooled_object
def reset(self, pooled_object: PooledObject, **kwargs: Any) -> PooledObject:
"""Reset the pooled object to a clean state."""
obj = pooled_object.keeped_object
# Reset the object by calling its reset method if it exists
if hasattr(obj, 'reset') and callable(getattr(obj, 'reset')):
obj.reset()
else:
# Fallback: clear all attributes to reset the object
if hasattr(obj, '__dict__'):
obj.__dict__.clear()
return pooled_object
def validate(self, pooled_object: PooledObject) -> bool:
"""Validate if the pooled object is still usable."""
return pooled_object.keeped_object is not None
# Global pond instances
_pools: dict[str, Pond] = {}
def get_object_pool(
pool_name: str,
object_class: Type[T],
pooled_maxsize: Optional[int] = None, # None = unlimited pool with eviction-based memory control
least_one: bool = True, # Always keep at least one
borrowed_timeout: int = 10, # Longer timeout for high concurrency
time_between_eviction_runs: int = 300, # Less frequent eviction to maintain high reuse ratio
eviction_weight: float = 0.3, # Less aggressive eviction for better reuse
prewarm_count: int = 5 # Lower pre-warm count to reduce initial memory usage
) -> Pond:
"""Get or create a global object pool instance with balanced eviction-based memory control.
Memory is controlled through moderate eviction to balance reuse ratio and memory usage:
- Moderate eviction frequency (300s) to maintain high object reuse ratio
- Conservative eviction weight (0.3) to avoid destroying useful objects
- Lower pre-warm count (5) to reduce initial memory footprint
Args:
pool_name: Unique name for the pool
object_class: The class type to pool
pooled_maxsize: Maximum number of objects in the pool (None = truly unlimited)
least_one: Whether to keep at least one object in the pool (default: True)
borrowed_timeout: Timeout for borrowing objects (seconds, default: 10)
time_between_eviction_runs: Time between eviction runs (seconds, default: 300)
eviction_weight: Weight for eviction algorithm (default: 0.3, conservative)
prewarm_count: Number of objects to pre-warm the pool with (default: 5)
Returns:
Pond instance for the specified object type
"""
if pool_name in _pools:
return _pools[pool_name]
# Create new pond
pond = Pond(
borrowed_timeout=borrowed_timeout,
time_between_eviction_runs=time_between_eviction_runs,
thread_daemon=True,
eviction_weight=eviction_weight
)
# Register the factory with user's maxsize preference
factory = GenericPooledObjectFactory(
object_class=object_class,
pooled_maxsize=pooled_maxsize,
least_one=least_one
)
pond.register(factory, name=f"{pool_name}Factory")
# Pre-warm the pool
_prewarm_pool(pond, pool_name, prewarm_count)
_pools[pool_name] = pond
return pond
def _prewarm_pool(pond: Pond, pool_name: str, prewarm_count: int = 20) -> None:
"""Pre-warm the pool with initial objects for high concurrency."""
for _ in range(prewarm_count):
try:
pooled_obj = pond.borrow(name=f"{pool_name}Factory")
pond.recycle(pooled_obj, name=f"{pool_name}Factory")
except Exception:
# If pre-warming fails, just continue
break
litellm/litellm_core_utils/prompt_templates/factory.py
+4 -7
View File
@@ -3079,7 +3079,6 @@ class BedrockConverseMessagesProcessor:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE)
return messages
@staticmethod
async def _bedrock_converse_messages_pt_async( # noqa: PLR0915
messages: List,
@@ -3124,9 +3123,9 @@ class BedrockConverseMessagesProcessor:
_part = BedrockContentBlock(text=element["text"])
_parts.append(_part)
elif element["type"] == "guarded_text":
# Wrap guarded_text in guardrailConverseContent block
# Wrap guarded_text in guardContent block
_part = BedrockContentBlock(
guardrailConverseContent={"text": element["text"]}
guardContent={"text": {"text": element["text"]}}
)
_parts.append(_part)
elif element["type"] == "image_url":
@@ -3171,7 +3170,6 @@ class BedrockConverseMessagesProcessor:
msg_i += 1
if user_content:
if len(contents) > 0 and contents[-1]["role"] == "user":
if (
assistant_continue_message is not None
@@ -3506,9 +3504,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
_part = BedrockContentBlock(text=element["text"])
_parts.append(_part)
elif element["type"] == "guarded_text":
# Wrap guarded_text in guardrailConverseContent block
# Wrap guarded_text in guardContent block
_part = BedrockContentBlock(
guardrailConverseContent={"text": element["text"]}
guardContent={"text": {"text": element["text"]}}
)
_parts.append(_part)
elif element["type"] == "image_url":
@@ -3554,7 +3552,6 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
msg_i += 1
if user_content:
if len(contents) > 0 and contents[-1]["role"] == "user":
if (
assistant_continue_message is not None
litellm/llms/anthropic/chat/transformation.py
+18 -1
View File
@@ -45,7 +45,10 @@ from litellm.types.llms.openai import (
OpenAIMcpServerTool,
OpenAIWebSearchOptions,
)
from litellm.types.utils import CompletionTokensDetailsWrapper
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.utils import (
@@ -820,12 +823,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
web_search_requests: Optional[int] = None
if (
"cache_creation_input_tokens" in _usage
and _usage["cache_creation_input_tokens"] is not None
):
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
prompt_tokens += cache_creation_input_tokens
if (
"cache_read_input_tokens" in _usage
and _usage["cache_read_input_tokens"] is not None
@@ -841,8 +846,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
int, _usage["server_tool_use"]["web_search_requests"]
)
if "cache_creation" in _usage and _usage["cache_creation"] is not None:
cache_creation_token_details = CacheCreationTokenDetails(
ephemeral_5m_input_tokens=_usage["cache_creation"].get(
"ephemeral_5m_input_tokens"
),
ephemeral_1h_input_tokens=_usage["cache_creation"].get(
"ephemeral_1h_input_tokens"
),
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_read_input_tokens,
cache_creation_token_details=cache_creation_token_details,
)
completion_token_details = (
CompletionTokensDetailsWrapper(
litellm/llms/bedrock/base_aws_llm.py
+147 -58
View File
@@ -20,7 +20,11 @@ from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL, BEDROCK_MAX_POLICY_SIZE
from litellm.constants import (
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
BEDROCK_INVOKE_PROVIDERS_LITERAL,
BEDROCK_MAX_POLICY_SIZE,
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.secret_managers.main import get_secret, get_secret_str
@@ -189,23 +193,32 @@ class BaseAWSLLM:
# Check if we're in IRSA and trying to assume the same role we already have
current_role_arn = os.getenv("AWS_ROLE_ARN")
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
# In IRSA environments, we should skip role assumption if we're already running as the target role
# This is true when:
# 1. We have AWS_ROLE_ARN set (current role)
# 2. We have AWS_WEB_IDENTITY_TOKEN_FILE set (IRSA environment)
# 3. The current role matches the requested role
if (current_role_arn and web_identity_token_file and
current_role_arn == aws_role_name):
verbose_logger.debug("Using IRSA same-role optimization: calling _auth_with_env_vars")
if (
current_role_arn
and web_identity_token_file
and current_role_arn == aws_role_name
):
verbose_logger.debug(
"Using IRSA same-role optimization: calling _auth_with_env_vars"
)
# We're already running as this role via IRSA, no need to assume it again
# Use the default boto3 credentials (which will use the IRSA credentials)
credentials, _cache_ttl = self._auth_with_env_vars()
else:
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
verbose_logger.debug(
"Using role assumption: calling _auth_with_aws_role"
)
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
aws_session_name = (
f"litellm-session-{int(datetime.now().timestamp())}"
)
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
@@ -318,6 +331,40 @@ class BaseAWSLLM:
return provider
return None
@staticmethod
def get_bedrock_embedding_provider(
model: str,
) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]:
"""
Helper function to get the bedrock embedding provider from the model
Handles scenarios like:
1. model=cohere.embed-english-v3:0 -> Returns `cohere`
2. model=amazon.titan-embed-text-v1 -> Returns `amazon`
3. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
4. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
"""
# Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
if "." in model:
parts = model.split(".")
# Check if the second part (after potential region) is a known provider
if len(parts) >= 2:
potential_provider = parts[1] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0"
if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
# Check if the first part is a known provider (standard format)
potential_provider = parts[0] # e.g., "cohere" from "cohere.embed-english-v3:0"
if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
# Fallback: check if any provider name appears in the model string
for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
if provider in model:
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider)
return None
def _get_aws_region_name(
self,
optional_params: dict,
@@ -479,55 +526,67 @@ class BaseAWSLLM:
iam_creds = session.get_credentials()
return iam_creds, self._get_default_ttl_for_boto3_credentials()
def _handle_irsa_cross_account(self, irsa_role_arn: str, aws_role_name: str,
aws_session_name: str, region: str, web_identity_token_file: str,
aws_external_id: Optional[str] = None) -> dict:
def _handle_irsa_cross_account(
self,
irsa_role_arn: str,
aws_role_name: str,
aws_session_name: str,
region: str,
web_identity_token_file: str,
aws_external_id: Optional[str] = None,
) -> dict:
"""Handle cross-account role assumption for IRSA."""
import boto3
verbose_logger.debug("Cross-account role assumption detected")
# Read the web identity token
with open(web_identity_token_file, 'r') as f:
with open(web_identity_token_file, "r") as f:
web_identity_token = f.read().strip()
# Create an STS client without credentials
with tracer.trace("boto3.client(sts) for manual IRSA"):
sts_client = boto3.client('sts', region_name=region)
sts_client = boto3.client("sts", region_name=region)
# Manually assume the IRSA role with the session name
verbose_logger.debug(f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}")
verbose_logger.debug(
f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}"
)
irsa_response = sts_client.assume_role_with_web_identity(
RoleArn=irsa_role_arn,
RoleSessionName=aws_session_name,
WebIdentityToken=web_identity_token
WebIdentityToken=web_identity_token,
)
# Extract the credentials from the IRSA assumption
irsa_creds = irsa_response["Credentials"]
# Create a new STS client with the IRSA credentials
with tracer.trace("boto3.client(sts) with manual IRSA credentials"):
sts_client_with_creds = boto3.client(
'sts',
"sts",
region_name=region,
aws_access_key_id=irsa_creds["AccessKeyId"],
aws_secret_access_key=irsa_creds["SecretAccessKey"],
aws_session_token=irsa_creds["SessionToken"]
aws_session_token=irsa_creds["SessionToken"],
)
# Get current caller identity for debugging
try:
caller_identity = sts_client_with_creds.get_caller_identity()
verbose_logger.debug(f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}")
verbose_logger.debug(
f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}"
)
except Exception as e:
verbose_logger.debug(f"Failed to get caller identity: {e}")
# Now assume the target role
verbose_logger.debug(f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}")
verbose_logger.debug(
f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}"
)
assume_role_params = {
"RoleArn": aws_role_name,
"RoleSessionName": aws_session_name
"RoleSessionName": aws_session_name,
}
# Add ExternalId parameter if provided
@@ -536,27 +595,36 @@ class BaseAWSLLM:
return sts_client_with_creds.assume_role(**assume_role_params)
def _handle_irsa_same_account(self, aws_role_name: str, aws_session_name: str, region: str,
aws_external_id: Optional[str] = None) -> dict:
def _handle_irsa_same_account(
self,
aws_role_name: str,
aws_session_name: str,
region: str,
aws_external_id: Optional[str] = None,
) -> dict:
"""Handle same-account role assumption for IRSA."""
import boto3
verbose_logger.debug("Same account role assumption, using automatic IRSA")
with tracer.trace("boto3.client(sts) with automatic IRSA"):
sts_client = boto3.client("sts", region_name=region)
# Get current caller identity for debugging
try:
caller_identity = sts_client.get_caller_identity()
verbose_logger.debug(f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}")
verbose_logger.debug(
f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}"
)
except Exception as e:
verbose_logger.debug(f"Failed to get caller identity: {e}")
# Assume the role
verbose_logger.debug(f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}")
verbose_logger.debug(
f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}"
)
assume_role_params = {
"RoleArn": aws_role_name,
"RoleSessionName": aws_session_name
"RoleSessionName": aws_session_name,
}
# Add ExternalId parameter if provided
@@ -565,20 +633,24 @@ class BaseAWSLLM:
return sts_client.assume_role(**assume_role_params)
def _extract_credentials_and_ttl(self, sts_response: dict) -> Tuple[Credentials, Optional[int]]:
def _extract_credentials_and_ttl(
self, sts_response: dict
) -> Tuple[Credentials, Optional[int]]:
"""Extract credentials and TTL from STS response."""
from botocore.credentials import Credentials
sts_credentials = sts_response["Credentials"]
credentials = Credentials(
access_key=sts_credentials["AccessKeyId"],
secret_key=sts_credentials["SecretAccessKey"],
token=sts_credentials["SessionToken"],
)
expiration_time = sts_credentials["Expiration"]
ttl = int((expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds())
ttl = int(
(expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds()
)
return credentials, ttl
@tracer.wrap()
@@ -600,34 +672,51 @@ class BaseAWSLLM:
# Check if we're in an EKS/IRSA environment
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
irsa_role_arn = os.getenv("AWS_ROLE_ARN")
# If we have IRSA environment variables and no explicit credentials,
# we need to use the web identity token flow
if (web_identity_token_file and irsa_role_arn and
aws_access_key_id is None and aws_secret_access_key is None):
if (
web_identity_token_file
and irsa_role_arn
and aws_access_key_id is None
and aws_secret_access_key is None
):
# For cross-account role assumption with specific session names,
# we need to manually assume the IRSA role first with the correct session name
verbose_logger.debug(f"IRSA detected: using web identity token from {web_identity_token_file}")
verbose_logger.debug(
f"IRSA detected: using web identity token from {web_identity_token_file}"
)
try:
# Get region from environment
region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
region = (
os.getenv("AWS_REGION")
or os.getenv("AWS_DEFAULT_REGION")
or "us-east-1"
)
# Check if we need to do cross-account role assumption
if aws_role_name != irsa_role_arn:
sts_response = self._handle_irsa_cross_account(
irsa_role_arn, aws_role_name, aws_session_name, region, web_identity_token_file, aws_external_id
irsa_role_arn,
aws_role_name,
aws_session_name,
region,
web_identity_token_file,
aws_external_id,
)
else:
sts_response = self._handle_irsa_same_account(
aws_role_name, aws_session_name, region, aws_external_id
)
return self._extract_credentials_and_ttl(sts_response)
except Exception as e:
verbose_logger.debug(f"Failed to assume role via IRSA: {e}")
if "AccessDenied" in str(e) and "is not authorized to perform: sts:AssumeRole" in str(e):
if "AccessDenied" in str(
e
) and "is not authorized to perform: sts:AssumeRole" in str(e):
# Provide a more helpful error message for trust policy issues
verbose_logger.error(
f"Access denied when trying to assume role {aws_role_name}. "
@@ -636,7 +725,7 @@ class BaseAWSLLM:
)
# Re-raise the exception instead of falling through
raise
# In EKS/IRSA environments, use ambient credentials (no explicit keys needed)
# This allows the web identity token to work automatically
if aws_access_key_id is None and aws_secret_access_key is None:
@@ -653,7 +742,7 @@ class BaseAWSLLM:
assume_role_params = {
"RoleArn": aws_role_name,
"RoleSessionName": aws_session_name
"RoleSessionName": aws_session_name,
}
# Add ExternalId parameter if provided
@@ -782,14 +871,14 @@ class BaseAWSLLM:
)
# Determine proxy_endpoint_url
if env_aws_bedrock_runtime_endpoint and isinstance(
env_aws_bedrock_runtime_endpoint, str
):
proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
elif aws_bedrock_runtime_endpoint is not None and isinstance(
if aws_bedrock_runtime_endpoint is not None and isinstance(
aws_bedrock_runtime_endpoint, str
):
proxy_endpoint_url = aws_bedrock_runtime_endpoint
elif env_aws_bedrock_runtime_endpoint and isinstance(
env_aws_bedrock_runtime_endpoint, str
):
proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
else:
proxy_endpoint_url = endpoint_url
litellm/llms/bedrock/chat/converse_transformation.py
+65
View File
@@ -102,6 +102,61 @@ class AmazonConverseConfig(BaseConfig):
"performanceConfig": PerformanceConfigBlock,
}
@staticmethod
def _convert_consecutive_user_messages_to_guarded_text(
messages: List[AllMessageValues], optional_params: dict
) -> List[AllMessageValues]:
"""
Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
and no guarded_text is already present in those messages.
"""
# Check if guardrailConfig is present
if "guardrailConfig" not in optional_params:
return messages
# Find all consecutive user messages at the end
consecutive_user_message_indices = []
for i in range(len(messages) - 1, -1, -1):
if messages[i].get("role") == "user":
consecutive_user_message_indices.append(i)
else:
break
if not consecutive_user_message_indices:
return messages
# Process each consecutive user message
messages_copy = copy.deepcopy(messages)
for user_message_index in consecutive_user_message_indices:
user_message = messages_copy[user_message_index]
content = user_message.get("content", [])
if isinstance(content, list):
has_guarded_text = any(
isinstance(item, dict) and item.get("type") == "guarded_text"
for item in content
)
if has_guarded_text:
continue # Skip this message if it already has guarded_text
# Convert text elements to guarded_text
new_content = []
for item in content:
if isinstance(item, dict) and item.get("type") == "text":
new_item = {"type": "guarded_text", "text": item["text"]} # type: ignore
new_content.append(new_item)
else:
new_content.append(item)
messages_copy[user_message_index]["content"] = new_content # type: ignore
elif isinstance(content, str):
# If content is a string, convert it to guarded_text
messages_copy[user_message_index]["content"] = [ # type: ignore
{"type": "guarded_text", "text": content} # type: ignore
]
return messages_copy
@classmethod
def get_config(cls):
return {
@@ -769,6 +824,11 @@ class AmazonConverseConfig(BaseConfig):
headers: Optional[dict] = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
messages, optional_params
)
## TRANSFORMATION ##
_data: CommonRequestObject = self._transform_request_helper(
@@ -821,6 +881,11 @@ class AmazonConverseConfig(BaseConfig):
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
messages, optional_params
)
_data: CommonRequestObject = self._transform_request_helper(
model=model,
system_content_blocks=system_content_blocks,
litellm/llms/bedrock/count_tokens/handler.py
+123
View File
@@ -0,0 +1,123 @@
"""
AWS Bedrock CountTokens API handler.
Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
"""
from typing import Any, Dict
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
class BedrockCountTokensHandler(BedrockCountTokensConfig):
"""
Simplified handler for AWS Bedrock CountTokens API requests.
Uses existing LiteLLM infrastructure for authentication and request handling.
"""
async def handle_count_tokens_request(
self,
request_data: Dict[str, Any],
litellm_params: Dict[str, Any],
resolved_model: str,
) -> Dict[str, Any]:
"""
Handle a CountTokens request using existing LiteLLM patterns.
Args:
request_data: The incoming request payload
litellm_params: LiteLLM configuration parameters
resolved_model: The actual model ID resolved from router
Returns:
Dictionary containing token count response
"""
try:
# Validate the request
self.validate_count_tokens_request(request_data)
verbose_logger.debug(
f"Processing CountTokens request for resolved model: {resolved_model}"
)
# Get AWS region using existing LiteLLM function
aws_region_name = self._get_aws_region_name(
optional_params=litellm_params,
model=resolved_model,
model_id=None,
)
verbose_logger.debug(f"Retrieved AWS region: {aws_region_name}")
# Transform request to Bedrock format (supports both Converse and InvokeModel)
bedrock_request = self.transform_anthropic_to_bedrock_count_tokens(
request_data=request_data
)
verbose_logger.debug(f"Transformed request: {bedrock_request}")
# Get endpoint URL using simplified function
endpoint_url = self.get_bedrock_count_tokens_endpoint(
resolved_model, aws_region_name
)
verbose_logger.debug(f"Making request to: {endpoint_url}")
# Use existing _sign_request method from BaseAWSLLM
headers = {"Content-Type": "application/json"}
signed_headers, signed_body = self._sign_request(
service_name="bedrock",
headers=headers,
optional_params=litellm_params,
request_data=bedrock_request,
api_base=endpoint_url,
model=resolved_model,
)
async_client = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK)
response = await async_client.post(
endpoint_url,
headers=signed_headers,
data=signed_body,
timeout=30.0,
)
verbose_logger.debug(f"Response status: {response.status_code}")
if response.status_code != 200:
error_text = response.text
verbose_logger.error(f"AWS Bedrock error: {error_text}")
raise HTTPException(
status_code=400,
detail={"error": f"AWS Bedrock error: {error_text}"},
)
bedrock_response = response.json()
verbose_logger.debug(f"Bedrock response: {bedrock_response}")
# Transform response back to expected format
final_response = self.transform_bedrock_response_to_anthropic(
bedrock_response
)
verbose_logger.debug(f"Final response: {final_response}")
return final_response
except HTTPException:
# Re-raise HTTP exceptions as-is
raise
except Exception as e:
verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
raise HTTPException(
status_code=500,
detail={"error": f"CountTokens processing error: {str(e)}"},
)
litellm/llms/bedrock/count_tokens/transformation.py
+213
View File
@@ -0,0 +1,213 @@
"""
AWS Bedrock CountTokens API transformation logic.
This module handles the transformation of requests from Anthropic Messages API format
to AWS Bedrock's CountTokens API format and vice versa.
"""
from typing import Any, Dict, List
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockModelInfo
class BedrockCountTokensConfig(BaseAWSLLM):
"""
Configuration and transformation logic for AWS Bedrock CountTokens API.
AWS Bedrock CountTokens API Specification:
- Endpoint: POST /model/{modelId}/count-tokens
- Input formats: 'invokeModel' or 'converse'
- Response: {"inputTokens": <number>}
"""
def _detect_input_type(self, request_data: Dict[str, Any]) -> str:
"""
Detect whether to use 'converse' or 'invokeModel' input format.
Args:
request_data: The original request data
Returns:
'converse' or 'invokeModel'
"""
# If the request has messages in the expected Anthropic format, use converse
if "messages" in request_data and isinstance(request_data["messages"], list):
return "converse"
# For raw text or other formats, use invokeModel
# This handles cases where the input is prompt-based or already in raw Bedrock format
return "invokeModel"
def transform_anthropic_to_bedrock_count_tokens(
self,
request_data: Dict[str, Any],
) -> Dict[str, Any]:
"""
Transform request to Bedrock CountTokens format.
Supports both Converse and InvokeModel input types.
Input (Anthropic format):
{
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "Hello!"}]
}
Output (Bedrock CountTokens format for Converse):
{
"input": {
"converse": {
"messages": [...],
"system": [...] (if present)
}
}
}
Output (Bedrock CountTokens format for InvokeModel):
{
"input": {
"invokeModel": {
"body": "{...raw model input...}"
}
}
}
"""
input_type = self._detect_input_type(request_data)
if input_type == "converse":
return self._transform_to_converse_format(request_data.get("messages", []))
else:
return self._transform_to_invoke_model_format(request_data)
def _transform_to_converse_format(
self, messages: List[Dict[str, Any]]
) -> Dict[str, Any]:
"""Transform to Converse input format."""
# Extract system messages if present
system_messages = []
user_messages = []
for message in messages:
if message.get("role") == "system":
system_messages.append({"text": message.get("content", "")})
else:
# Transform message content to Bedrock format
transformed_message: Dict[str, Any] = {"role": message.get("role"), "content": []}
# Handle content - ensure it's in the correct array format
content = message.get("content", "")
if isinstance(content, str):
# String content -> convert to text block
transformed_message["content"].append({"text": content})
elif isinstance(content, list):
# Already in blocks format - use as is
transformed_message["content"] = content
user_messages.append(transformed_message)
# Build the converse input format
converse_input = {"messages": user_messages}
# Add system messages if present
if system_messages:
converse_input["system"] = system_messages
# Build the complete request
return {"input": {"converse": converse_input}}
def _transform_to_invoke_model_format(
self, request_data: Dict[str, Any]
) -> Dict[str, Any]:
"""Transform to InvokeModel input format."""
import json
# For InvokeModel, we need to provide the raw body that would be sent to the model
# Remove the 'model' field from the body as it's not part of the model input
body_data = {k: v for k, v in request_data.items() if k != "model"}
return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
def get_bedrock_count_tokens_endpoint(
self, model: str, aws_region_name: str
) -> str:
"""
Construct the AWS Bedrock CountTokens API endpoint using existing LiteLLM functions.
Args:
model: The resolved model ID from router lookup
aws_region_name: AWS region (e.g., "eu-west-1")
Returns:
Complete endpoint URL for CountTokens API
"""
# Use existing LiteLLM function to get the base model ID (removes region prefix)
model_id = BedrockModelInfo.get_base_model(model)
# Remove bedrock/ prefix if present
if model_id.startswith("bedrock/"):
model_id = model_id[8:] # Remove "bedrock/" prefix
base_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
endpoint = f"{base_url}/model/{model_id}/count-tokens"
return endpoint
def transform_bedrock_response_to_anthropic(
self, bedrock_response: Dict[str, Any]
) -> Dict[str, Any]:
"""
Transform Bedrock CountTokens response to Anthropic format.
Input (Bedrock response):
{
"inputTokens": 123
}
Output (Anthropic format):
{
"input_tokens": 123
}
"""
input_tokens = bedrock_response.get("inputTokens", 0)
return {"input_tokens": input_tokens}
def validate_count_tokens_request(self, request_data: Dict[str, Any]) -> None:
"""
Validate the incoming count tokens request.
Supports both Converse and InvokeModel input formats.
Args:
request_data: The request payload
Raises:
ValueError: If the request is invalid
"""
if not request_data.get("model"):
raise ValueError("model parameter is required")
input_type = self._detect_input_type(request_data)
if input_type == "converse":
# Validate Converse format (messages-based)
messages = request_data.get("messages", [])
if not messages:
raise ValueError("messages parameter is required for Converse input")
if not isinstance(messages, list):
raise ValueError("messages must be a list")
for i, message in enumerate(messages):
if not isinstance(message, dict):
raise ValueError(f"Message {i} must be a dictionary")
if "role" not in message:
raise ValueError(f"Message {i} must have a 'role' field")
if "content" not in message:
raise ValueError(f"Message {i} must have a 'content' field")
else:
# For InvokeModel format, we need at least some content to count tokens
# The content structure varies by model, so we do minimal validation
if len(request_data) <= 1: # Only has 'model' field
raise ValueError("Request must contain content to count tokens")
litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py
+37 -15
View File
@@ -10,7 +10,7 @@ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-tit
"""
import types
from typing import List, Optional
from typing import List, Optional, Union
from litellm.types.llms.bedrock import (
AmazonTitanV2EmbeddingRequest,
@@ -30,9 +30,7 @@ class AmazonTitanV2Config:
normalize: Optional[bool] = None
dimensions: Optional[int] = None
def __init__(
self, normalize: Optional[bool] = None, dimensions: Optional[int] = None
) -> None:
def __init__(self, normalize: Optional[bool] = None, dimensions: Optional[int] = None) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
@@ -57,32 +55,56 @@ class AmazonTitanV2Config:
}
def get_supported_openai_params(self) -> List[str]:
return ["dimensions"]
return ["dimensions", "encoding_format"]
def map_openai_params(
self, non_default_params: dict, optional_params: dict
) -> dict:
def map_openai_params(self, non_default_params: dict, optional_params: dict) -> dict:
for k, v in non_default_params.items():
if k == "dimensions":
optional_params["dimensions"] = v
elif k == "encoding_format":
# Map OpenAI encoding_format to AWS embeddingTypes
if v == "float":
optional_params["embeddingTypes"] = ["float"]
elif v == "base64":
# base64 maps to binary format in AWS
optional_params["embeddingTypes"] = ["binary"]
else:
# For any other encoding format, default to float
optional_params["embeddingTypes"] = ["float"]
return optional_params
def _transform_request(
self, input: str, inference_params: dict
) -> AmazonTitanV2EmbeddingRequest:
def _transform_request(self, input: str, inference_params: dict) -> AmazonTitanV2EmbeddingRequest:
return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params) # type: ignore
def _transform_response(
self, response_list: List[dict], model: str
) -> EmbeddingResponse:
def _transform_response(self, response_list: List[dict], model: str) -> EmbeddingResponse:
total_prompt_tokens = 0
transformed_responses: List[Embedding] = []
for index, response in enumerate(response_list):
_parsed_response = AmazonTitanV2EmbeddingResponse(**response) # type: ignore
# According to AWS docs, embeddingsByType is always present
# If binary was requested (encoding_format="base64"), use binary data
# Otherwise, use float data from embeddingsByType or fallback to embedding field
embedding_data: Union[List[float], List[int]]
if ("embeddingsByType" in _parsed_response and
"binary" in _parsed_response["embeddingsByType"]):
# Use binary data if available (for encoding_format="base64")
embedding_data = _parsed_response["embeddingsByType"]["binary"]
elif ("embeddingsByType" in _parsed_response and
"float" in _parsed_response["embeddingsByType"]):
# Use float data from embeddingsByType
embedding_data = _parsed_response["embeddingsByType"]["float"]
elif "embedding" in _parsed_response:
# Fallback to legacy embedding field
embedding_data = _parsed_response["embedding"]
else:
raise ValueError(f"No embedding data found in response: {response}")
transformed_responses.append(
Embedding(
embedding=_parsed_response["embedding"],
embedding=embedding_data,
index=index,
object="embedding",
)
litellm/llms/bedrock/embed/embedding.py
+95 -76
View File
@@ -4,12 +4,13 @@ Handles embedding calls to Bedrock's `/invoke` endpoint
import copy
import json
from typing import Any, Callable, List, Optional, Tuple, Union
import urllib.parse
from typing import Any, Callable, List, Optional, Tuple, Union, get_args
import httpx
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@@ -18,7 +19,11 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
)
from litellm.secret_managers.main import get_secret
from litellm.types.llms.bedrock import AmazonEmbeddingRequest, CohereEmbeddingRequest
from litellm.types.llms.bedrock import (
AmazonEmbeddingRequest,
CohereEmbeddingRequest,
TwelveLabsMarengoEmbeddingRequest,
)
from litellm.types.utils import EmbeddingResponse
from ..base_aws_llm import BaseAWSLLM
@@ -29,6 +34,7 @@ from .amazon_titan_multimodal_transformation import (
)
from .amazon_titan_v2_transformation import AmazonTitanV2Config
from .cohere_transformation import BedrockCohereEmbeddingConfig
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
class BedrockEmbedding(BaseAWSLLM):
@@ -145,6 +151,44 @@ class BedrockEmbedding(BaseAWSLLM):
raise BedrockError(status_code=408, message="Timeout error occurred.")
return response.json()
def _transform_response(
self, response_list: List[dict], model: str, provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL
) -> Optional[EmbeddingResponse]:
"""
Transforms the response from the Bedrock embedding provider to the OpenAI format.
"""
returned_response: Optional[EmbeddingResponse] = None
if model == "amazon.titan-embed-image-v1":
returned_response = (
AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
response_list=response_list, model=model
)
)
elif model == "amazon.titan-embed-text-v1":
returned_response = AmazonTitanG1Config()._transform_response(
response_list=response_list, model=model
)
elif model == "amazon.titan-embed-text-v2:0":
returned_response = AmazonTitanV2Config()._transform_response(
response_list=response_list, model=model
)
elif provider == "twelvelabs":
returned_response = TwelveLabsMarengoEmbeddingConfig()._transform_response(
response_list=response_list, model=model
)
##########################################################
# Validate returned response
##########################################################
if returned_response is None:
raise Exception(
"Unable to map model response to known provider format. model={}".format(
model
)
)
return returned_response
def _single_func_embeddings(
self,
@@ -157,6 +201,7 @@ class BedrockEmbedding(BaseAWSLLM):
aws_region_name: str,
model: str,
logging_obj: Any,
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
api_key: Optional[str] = None,
):
responses: List[dict] = []
@@ -164,16 +209,16 @@ class BedrockEmbedding(BaseAWSLLM):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=json.dumps(data),
headers=headers,
api_key=api_key
)
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=json.dumps(data),
headers=headers,
api_key=api_key,
)
## LOGGING
logging_obj.pre_call(
@@ -203,32 +248,9 @@ class BedrockEmbedding(BaseAWSLLM):
responses.append(response)
returned_response: Optional[EmbeddingResponse] = None
## TRANSFORM RESPONSE ##
if model == "amazon.titan-embed-image-v1":
returned_response = (
AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
response_list=responses, model=model
)
)
elif model == "amazon.titan-embed-text-v1":
returned_response = AmazonTitanG1Config()._transform_response(
response_list=responses, model=model
)
elif model == "amazon.titan-embed-text-v2:0":
returned_response = AmazonTitanV2Config()._transform_response(
response_list=responses, model=model
)
if returned_response is None:
raise Exception(
"Unable to map model response to known provider format. model={}".format(
model
)
)
return returned_response
return self._transform_response(
response_list=responses, model=model, provider=provider
)
async def _async_single_func_embeddings(
self,
@@ -241,6 +263,7 @@ class BedrockEmbedding(BaseAWSLLM):
aws_region_name: str,
model: str,
logging_obj: Any,
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
api_key: Optional[str] = None,
):
responses: List[dict] = []
@@ -248,16 +271,16 @@ class BedrockEmbedding(BaseAWSLLM):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=json.dumps(data),
headers=headers,
api_key=api_key,
)
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=json.dumps(data),
headers=headers,
api_key=api_key,
)
## LOGGING
logging_obj.pre_call(
@@ -286,33 +309,10 @@ class BedrockEmbedding(BaseAWSLLM):
)
responses.append(response)
returned_response: Optional[EmbeddingResponse] = None
## TRANSFORM RESPONSE ##
if model == "amazon.titan-embed-image-v1":
returned_response = (
AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
response_list=responses, model=model
)
)
elif model == "amazon.titan-embed-text-v1":
returned_response = AmazonTitanG1Config()._transform_response(
response_list=responses, model=model
)
elif model == "amazon.titan-embed-text-v2:0":
returned_response = AmazonTitanV2Config()._transform_response(
response_list=responses, model=model
)
if returned_response is None:
raise Exception(
"Unable to map model response to known provider format. model={}".format(
model
)
)
return returned_response
return self._transform_response(
response_list=responses, model=model, provider=provider
)
def embeddings(
self,
@@ -336,7 +336,7 @@ class BedrockEmbedding(BaseAWSLLM):
### TRANSFORMATION ###
unencoded_model_id = (
optional_params.pop("model_id", None) or model
) # default to model if not passed
) # default to model if not passed
modelId = urllib.parse.quote(unencoded_model_id, safe="")
aws_region_name = self._get_aws_region_name(
optional_params=optional_params,
@@ -344,7 +344,12 @@ class BedrockEmbedding(BaseAWSLLM):
model_id=unencoded_model_id,
)
provider = model.split(".")[0]
provider = self.get_bedrock_embedding_provider(model)
if provider is None:
raise Exception(
f"Unable to determine bedrock embedding provider for model: {model}. "
f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
)
inference_params = copy.deepcopy(optional_params)
inference_params = {
k: v
@@ -394,6 +399,15 @@ class BedrockEmbedding(BaseAWSLLM):
)
)
batch_data.append(transformed_request)
elif provider == "twelvelabs":
batch_data = []
for i in input:
twelvelabs_request: (
TwelveLabsMarengoEmbeddingRequest
) = TwelveLabsMarengoEmbeddingConfig()._transform_request(
input=i, inference_params=inference_params
)
batch_data.append(twelvelabs_request)
### SET RUNTIME ENDPOINT ###
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
@@ -422,8 +436,9 @@ class BedrockEmbedding(BaseAWSLLM):
model=model,
logging_obj=logging_obj,
api_key=api_key,
provider=provider,
)
return self._single_func_embeddings(
returned_response = self._single_func_embeddings(
client=(
client
if client is not None and isinstance(client, HTTPHandler)
@@ -438,14 +453,18 @@ class BedrockEmbedding(BaseAWSLLM):
model=model,
logging_obj=logging_obj,
api_key=api_key,
provider=provider,
)
if returned_response is None:
raise Exception("Unable to map Bedrock request to provider")
return returned_response
elif data is None:
raise Exception("Unable to map Bedrock request to provider")
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=credentials,
aws_region_name=aws_region_name,
litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py
+140
View File
@@ -0,0 +1,140 @@
"""
Transformation logic from OpenAI /v1/embeddings format to Bedrock TwelveLabs Marengo /invoke format.
Why separate file? Make it easy to see how transformation works
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
"""
from typing import List
from litellm.types.llms.bedrock import (
TwelveLabsMarengoEmbeddingRequest,
)
from litellm.types.utils import Embedding, EmbeddingResponse, Usage
from litellm.utils import get_base64_str, is_base64_encoded
class TwelveLabsMarengoEmbeddingConfig:
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
Supports text and image inputs for Phase 1.
Video and audio support will be added in Phase 2.
"""
def __init__(self) -> None:
pass
def get_supported_openai_params(self) -> List[str]:
return ["encoding_format", "textTruncate", "embeddingOption"]
def map_openai_params(
self, non_default_params: dict, optional_params: dict
) -> dict:
for k, v in non_default_params.items():
if k == "encoding_format":
# TwelveLabs doesn't have encoding_format, but we can map it to embeddingOption
if v == "float":
optional_params["embeddingOption"] = ["visual-text", "visual-image"]
elif k == "textTruncate":
optional_params["textTruncate"] = v
elif k == "embeddingOption":
optional_params["embeddingOption"] = v
return optional_params
def _transform_request(
self, input: str, inference_params: dict
) -> TwelveLabsMarengoEmbeddingRequest:
"""
Transform OpenAI-style input to TwelveLabs Marengo format.
Phase 1: Supports text and image inputs only.
"""
# Check if input is base64 encoded image
is_encoded = is_base64_encoded(input)
if is_encoded:
# Image input
b64_str = get_base64_str(input)
transformed_request = TwelveLabsMarengoEmbeddingRequest(
inputType="image", mediaSource={"base64String": b64_str}
)
else:
# Text input
transformed_request = TwelveLabsMarengoEmbeddingRequest(
inputType="text", inputText=input
)
# Set default textTruncate if not specified
if "textTruncate" not in inference_params:
transformed_request["textTruncate"] = "end"
# Apply any additional inference parameters
for k, v in inference_params.items():
if k not in [
"inputType",
"inputText",
"mediaSource",
]: # Don't override core fields
transformed_request[k] = v # type: ignore
return transformed_request
def _transform_response(
self, response_list: List[dict], model: str
) -> EmbeddingResponse:
"""
Transform TwelveLabs response to OpenAI format.
Handles the actual TwelveLabs response format: {"data": [{"embedding": [...]}]}
"""
embeddings: List[Embedding] = []
total_tokens = 0
for response in response_list:
# TwelveLabs response format has a "data" field containing the embeddings
if "data" in response and isinstance(response["data"], list):
for item in response["data"]:
if "embedding" in item:
# Single embedding response
embedding = Embedding(
embedding=item["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
# Estimate token count (rough approximation)
if "inputTextTokenCount" in item:
total_tokens += item["inputTextTokenCount"]
else:
# Rough estimate: 1 token per 4 characters for text, or use embedding size
total_tokens += len(item["embedding"]) // 4
elif "embedding" in response:
# Direct embedding response (fallback for other formats)
embedding = Embedding(
embedding=response["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
# Estimate token count (rough approximation)
if "inputTextTokenCount" in response:
total_tokens += response["inputTextTokenCount"]
else:
# Rough estimate: 1 token per 4 characters for text
total_tokens += len(response.get("inputText", "")) // 4
elif "embeddings" in response:
# Multiple embeddings response (from video/audio)
for i, emb in enumerate(response["embeddings"]):
embedding = Embedding(
embedding=emb["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
total_tokens += len(emb["embedding"]) // 4 # Rough estimate
usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
return EmbeddingResponse(data=embeddings, model=model, usage=usage)
litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py
+7 -2
View File
@@ -7,12 +7,12 @@ from litellm.types.llms.bedrock import (
AmazonNovaCanvasColorGuidedGenerationParams,
AmazonNovaCanvasColorGuidedRequest,
AmazonNovaCanvasImageGenerationConfig,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
AmazonNovaCanvasRequestBase,
AmazonNovaCanvasTextToImageParams,
AmazonNovaCanvasTextToImageRequest,
AmazonNovaCanvasTextToImageResponse,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
)
from litellm.types.utils import ImageResponse
@@ -67,6 +67,11 @@ class AmazonNovaCanvasConfig:
"""
task_type = optional_params.pop("taskType", "TEXT_IMAGE")
image_generation_config = optional_params.pop("imageGenerationConfig", {})
# Extract model_id parameter to prevent "extraneous key" error from Bedrock API
# Following the same pattern as chat completions and embeddings
unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
image_generation_config = {**image_generation_config, **optional_params}
if task_type == "TEXT_IMAGE":
text_to_image_params: Dict[str, Any] = image_generation_config.pop(
litellm/llms/bedrock/image/image_handler.py
+11 -1
View File
@@ -233,7 +233,17 @@ class BedrockImageGeneration(BaseAWSLLM):
Returns:
dict: The request body to use for the Bedrock Image Generation API
"""
provider = model.split(".")[0]
# Use the existing ARN-aware provider detection method
bedrock_provider = self.get_bedrock_invoke_provider(model)
if bedrock_provider == "amazon" or bedrock_provider == "nova":
# Handle Amazon Nova Canvas models
provider = "amazon"
elif bedrock_provider == "stability":
provider = "stability"
else:
# Fallback to original logic for backward compatibility
provider = model.split(".")[0]
inference_params = copy.deepcopy(optional_params)
inference_params.pop(
"user", None
litellm/llms/cohere/completion/handler.py
-5
View File
@@ -1,5 +0,0 @@
"""
Cohere /generate API - uses `llm_http_handler.py` to make httpx requests
Request/Response transformation is handled in `transformation.py`
"""
litellm/llms/cohere/completion/transformation.py
-265
View File
@@ -1,265 +0,0 @@
import time
from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from ..common_utils import CohereError
from ..common_utils import ModelResponseIterator as CohereModelResponseIterator
from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class CohereTextConfig(BaseConfig):
"""
Reference: https://docs.cohere.com/reference/generate
The class `CohereConfig` provides configuration for the Cohere's API interface. Below are the parameters:
- `num_generations` (integer): Maximum number of generations returned. Default is 1, with a minimum value of 1 and a maximum value of 5.
- `max_tokens` (integer): Maximum number of tokens the model will generate as part of the response. Default value is 20.
- `truncate` (string): Specifies how the API handles inputs longer than maximum token length. Options include NONE, START, END. Default is END.
- `temperature` (number): A non-negative float controlling the randomness in generation. Lower temperatures result in less random generations. Default is 0.75.
- `preset` (string): Identifier of a custom preset, a combination of parameters such as prompt, temperature etc.
- `end_sequences` (array of strings): The generated text gets cut at the beginning of the earliest occurrence of an end sequence, which will be excluded from the text.
- `stop_sequences` (array of strings): The generated text gets cut at the end of the earliest occurrence of a stop sequence, which will be included in the text.
- `k` (integer): Limits generation at each step to top `k` most likely tokens. Default is 0.
- `p` (number): Limits generation at each step to most likely tokens with total probability mass of `p`. Default is 0.
- `frequency_penalty` (number): Reduces repetitiveness of generated tokens. Higher values apply stronger penalties to previously occurred tokens.
- `presence_penalty` (number): Reduces repetitiveness of generated tokens. Similar to frequency_penalty, but this penalty applies equally to all tokens that have already appeared.
- `return_likelihoods` (string): Specifies how and if token likelihoods are returned with the response. Options include GENERATION, ALL and NONE.
- `logit_bias` (object): Used to prevent the model from generating unwanted tokens or to incentivize it to include desired tokens. e.g. {"hello_world": 1233}
"""
num_generations: Optional[int] = None
max_tokens: Optional[int] = None
truncate: Optional[str] = None
temperature: Optional[int] = None
preset: Optional[str] = None
end_sequences: Optional[list] = None
stop_sequences: Optional[list] = None
k: Optional[int] = None
p: Optional[int] = None
frequency_penalty: Optional[int] = None
presence_penalty: Optional[int] = None
return_likelihoods: Optional[str] = None
logit_bias: Optional[dict] = None
def __init__(
self,
num_generations: Optional[int] = None,
max_tokens: Optional[int] = None,
truncate: Optional[str] = None,
temperature: Optional[int] = None,
preset: Optional[str] = None,
end_sequences: Optional[list] = None,
stop_sequences: Optional[list] = None,
k: Optional[int] = None,
p: Optional[int] = None,
frequency_penalty: Optional[int] = None,
presence_penalty: Optional[int] = None,
return_likelihoods: Optional[str] = None,
logit_bias: Optional[dict] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@classmethod
def get_config(cls):
return super().get_config()
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
return cohere_validate_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
api_key=api_key,
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
return CohereError(status_code=status_code, message=error_message)
def get_supported_openai_params(self, model: str) -> List:
return [
"stream",
"temperature",
"max_tokens",
"logit_bias",
"top_p",
"frequency_penalty",
"presence_penalty",
"stop",
"n",
"extra_headers",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
for param, value in non_default_params.items():
if param == "stream":
optional_params["stream"] = value
elif param == "temperature":
optional_params["temperature"] = value
elif param == "max_tokens":
optional_params["max_tokens"] = value
elif param == "n":
optional_params["num_generations"] = value
elif param == "logit_bias":
optional_params["logit_bias"] = value
elif param == "top_p":
optional_params["p"] = value
elif param == "frequency_penalty":
optional_params["frequency_penalty"] = value
elif param == "presence_penalty":
optional_params["presence_penalty"] = value
elif param == "stop":
optional_params["stop_sequences"] = value
return optional_params
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
prompt = " ".join(
convert_content_list_to_str(message=message) for message in messages
)
## Load Config
config = litellm.CohereConfig.get_config()
for k, v in config.items():
if (
k not in optional_params
): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
## Handle Tool Calling
if "tools" in optional_params:
_is_function_call = True
tool_calling_system_prompt = self._construct_cohere_tool_for_completion_api(
tools=optional_params["tools"]
)
optional_params["tools"] = tool_calling_system_prompt
data = {
"model": model,
"prompt": prompt,
**optional_params,
}
return data
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
prompt = " ".join(
convert_content_list_to_str(message=message) for message in messages
)
completion_response = raw_response.json()
choices_list = []
for idx, item in enumerate(completion_response["generations"]):
if len(item["text"]) > 0:
message_obj = Message(content=item["text"])
else:
message_obj = Message(content=None)
choice_obj = Choices(
finish_reason=item["finish_reason"],
index=idx + 1,
message=message_obj,
)
choices_list.append(choice_obj)
model_response.choices = choices_list # type: ignore
## CALCULATING USAGE
prompt_tokens = len(encoding.encode(prompt))
completion_tokens = len(
encoding.encode(model_response["choices"][0]["message"].get("content", ""))
)
model_response.created = int(time.time())
model_response.model = model
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
setattr(model_response, "usage", usage)
return model_response
def _construct_cohere_tool_for_completion_api(
self,
tools: Optional[List] = None,
) -> dict:
if tools is None:
tools = []
return {"tools": tools}
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
):
return CohereModelResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
litellm/llms/gemini/google_genai/transformation.py
+2 -1
View File
@@ -121,7 +121,8 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
default_headers = {
"Content-Type": "application/json",
}
gemini_api_key = self._get_google_ai_studio_api_key(dict(litellm_params or {}))
# Use the passed api_key first, then fall back to litellm_params and environment
gemini_api_key = api_key or self._get_google_ai_studio_api_key(dict(litellm_params or {}))
if gemini_api_key is not None:
default_headers[self.XGOOGLE_API_KEY] = gemini_api_key
if headers is not None:
litellm/llms/gemini/image_generation/transformation.py
+78 -37
View File
@@ -85,17 +85,25 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
) -> str:
"""
Get the complete url for the request
Google AI API format: https://generativelanguage.googleapis.com/v1beta/models/{model}:predict
Gemini 2.5 Flash Image Preview: :generateContent
Other Imagen models: :predict
"""
complete_url: str = (
api_base
or get_secret_str("GEMINI_API_BASE")
api_base
or get_secret_str("GEMINI_API_BASE")
or self.DEFAULT_BASE_URL
)
complete_url = complete_url.rstrip("/")
complete_url = f"{complete_url}/models/{model}:predict"
# Gemini 2.5 Flash Image Preview uses generateContent endpoint
if "2.5-flash-image-preview" in model:
complete_url = f"{complete_url}/models/{model}:generateContent"
else:
# All other Imagen models use predict endpoint
complete_url = f"{complete_url}/models/{model}:predict"
return complete_url
def validate_environment(
@@ -128,35 +136,52 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
headers: dict,
) -> dict:
"""
Transform the image generation request to Google AI Imagen format
Google AI API format:
Transform the image generation request to Gemini format
For Gemini 2.5 Flash Image Preview, use the standard Gemini format with response_modalities:
{
"instances": [
"contents": [
{
"prompt": "Robot holding a red skateboard"
"parts": [
{"text": "Generate an image of..."}
]
}
],
"parameters": {
"sampleCount": 4,
"aspectRatio": "1:1",
"personGeneration": "allow_adult"
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
"""
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body.model_dump(exclude_none=True)
# For Gemini 2.5 Flash Image Preview, use standard Gemini format
if "2.5-flash-image-preview" in model:
request_body: dict = {
"contents": [
{
"parts": [
{"text": prompt}
]
}
],
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
return request_body
else:
# For other Imagen models, use the original Imagen format
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body_obj: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body_obj.model_dump(exclude_none=True)
def transform_image_generation_response(
self,
@@ -185,14 +210,30 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
if not model_response.data:
model_response.data = []
# Google AI returns predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
# Handle different response formats based on model
if "2.5-flash-image-preview" in model:
# Gemini 2.5 Flash Image Preview returns in candidates format
candidates = response_data.get("candidates", [])
for candidate in candidates:
content = candidate.get("content", {})
parts = content.get("parts", [])
for part in parts:
# Look for inlineData with image
if "inlineData" in part:
inline_data = part["inlineData"]
if "data" in inline_data:
model_response.data.append(ImageObject(
b64_json=inline_data["data"],
url=None,
))
else:
# Original Imagen format - predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
return model_response
litellm/llms/vertex_ai/batches/transformation.py
+8 -1
View File
@@ -114,7 +114,14 @@ class VertexAIBatchTransformation:
"""
Gets the output file id from the Vertex AI Batch response
"""
output_file_id: str = ""
output_file_id: str = (
response.get("outputInfo", OutputInfo()).get("gcsOutputDirectory", "")
+ "/predictions.jsonl"
)
if output_file_id != "/predictions.jsonl":
return output_file_id
output_config = response.get("outputConfig")
if output_config is None:
return output_file_id
litellm/llms/vertex_ai/files/handler.py
+141 -2
View File
@@ -1,5 +1,6 @@
import asyncio
from typing import Any, Coroutine, Optional, Union
import urllib.parse
from typing import Any, Coroutine, Optional, Tuple, Union
import httpx
@@ -9,7 +10,12 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import (
GCSLoggingConfig,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import CreateFileRequest, OpenAIFileObject
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAIFileObject,
)
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from .transformation import VertexAIJsonlFilesTransformation
@@ -105,3 +111,136 @@ class VertexAIFilesHandler(GCSBucketBase):
max_retries=max_retries,
)
)
def _extract_bucket_and_object_from_file_id(self, file_id: str) -> Tuple[str, str]:
"""
Extract bucket name and object path from URL-encoded file_id.
Expected format: gs%3A%2F%2Fbucket-name%2Fpath%2Fto%2Ffile
Which decodes to: gs://bucket-name/path/to/file
Returns:
tuple: (bucket_name, url_encoded_object_path)
- bucket_name: "bucket-name"
- url_encoded_object_path: "path%2Fto%2Ffile"
"""
decoded_path = urllib.parse.unquote(file_id)
if decoded_path.startswith("gs://"):
full_path = decoded_path[5:] # Remove 'gs://' prefix
else:
full_path = decoded_path
if "/" in full_path:
bucket_name, object_path = full_path.split("/", 1)
else:
bucket_name = full_path
object_path = ""
encoded_object_path = urllib.parse.quote(object_path, safe="")
return bucket_name, encoded_object_path
async def afile_content(
self,
file_content_request: FileContentRequest,
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> HttpxBinaryResponseContent:
"""
Download file content from GCS bucket for VertexAI files.
Args:
file_content_request: Contains file_id (URL-encoded GCS path)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
"""
file_id = file_content_request.get("file_id")
if not file_id:
raise ValueError("file_id is required in file_content_request")
bucket_name, encoded_object_path = self._extract_bucket_and_object_from_file_id(
file_id
)
download_kwargs = {
"standard_callback_dynamic_params": {"gcs_bucket_name": bucket_name}
}
file_content = await self.download_gcs_object(
object_name=encoded_object_path, **download_kwargs
)
if file_content is None:
decoded_path = urllib.parse.unquote(file_id)
raise ValueError(f"Failed to download file from GCS: {decoded_path}")
decoded_path = urllib.parse.unquote(file_id)
mock_response = httpx.Response(
status_code=200,
content=file_content,
headers={"content-type": "application/octet-stream"},
request=httpx.Request(method="GET", url=decoded_path),
)
return HttpxBinaryResponseContent(response=mock_response)
def file_content(
self,
_is_async: bool,
file_content_request: FileContentRequest,
api_base: Optional[str],
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> Union[
HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
]:
"""
Download file content from GCS bucket for VertexAI files.
Supports both sync and async operations.
Args:
_is_async: Whether to run asynchronously
file_content_request: Contains file_id (URL-encoded GCS path)
api_base: API base (unused for GCS operations)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
"""
if _is_async:
return self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
else:
return asyncio.run(
self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
)
litellm/llms/vertex_ai/files/transformation.py
+3 -3
View File
@@ -261,10 +261,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
extracted_file_data = extract_file_data(file_data)
extracted_file_data_content = extracted_file_data.get("content")
if extracted_file_data_content is None:
raise ValueError("file content is required")
if FilesAPIUtils.is_batch_jsonl_file(
create_file_data=create_file_data,
extracted_file_data=extracted_file_data,
@@ -283,7 +283,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
openai_jsonl_content
)
)
return json.dumps(vertex_jsonl_content)
return "\n".join(json.dumps(item) for item in vertex_jsonl_content)
elif isinstance(extracted_file_data_content, bytes):
return extracted_file_data_content
else:
litellm/main.py
+5 -47
View File
@@ -2395,43 +2395,7 @@ def completion( # type: ignore # noqa: PLR0915
)
return response
response = model_response
elif custom_llm_provider == "cohere":
cohere_key = (
api_key
or litellm.cohere_key
or get_secret("COHERE_API_KEY")
or get_secret("CO_API_KEY")
or litellm.api_key
)
api_base = (
api_base
or litellm.api_base
or get_secret("COHERE_API_BASE")
or "https://api.cohere.ai/v1/generate"
)
# Use the consolidated headers that were already merged at the top of the function
# No need for additional merging here as it's already done
response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider="cohere",
timeout=timeout,
headers=headers,
encoding=encoding,
api_key=cohere_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
elif custom_llm_provider == "cohere_chat":
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
cohere_key = (
api_key
or litellm.cohere_key
@@ -2447,12 +2411,8 @@ def completion( # type: ignore # noqa: PLR0915
or "https://api.cohere.ai/v1/chat"
)
headers = headers or litellm.headers or {}
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
# Use the consolidated headers that were already merged at the top of the function
# No need for additional merging here as it's already done
response = base_llm_http_handler.completion(
model=model,
@@ -4177,10 +4137,8 @@ def embedding( # noqa: PLR0915
or litellm.api_key
)
if extra_headers is not None and isinstance(extra_headers, dict):
headers = extra_headers
else:
headers = {}
# Use the consolidated headers that were already merged at the top of the function
# No need for additional merging here as it's already done
response = base_llm_http_handler.embedding(
model=model,
litellm/model_prices_and_context_window_backup.json
+62 -2
View File
@@ -296,6 +296,66 @@
"output_cost_per_token": 0.0,
"output_vector_size": 1024
},
"twelvelabs.marengo-embed-2-7-v1:0": {
"input_cost_per_token": 7e-05,
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"max_tokens": 77,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"supports_embedding_image_input": true,
"supports_image_input": true
},
"us.twelvelabs.marengo-embed-2-7-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"max_tokens": 77,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"supports_embedding_image_input": true,
"supports_image_input": true
},
"eu.twelvelabs.marengo-embed-2-7-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"max_tokens": 77,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"supports_embedding_image_input": true,
"supports_image_input": true
},
"twelvelabs.pegasus-1-2-v1:0": {
"input_cost_per_video_per_second": 0.00049,
"output_cost_per_token": 7.5e-06,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_video_input": true
},
"us.twelvelabs.pegasus-1-2-v1:0": {
"input_cost_per_video_per_second": 0.00049,
"output_cost_per_token": 7.5e-06,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_video_input": true
},
"eu.twelvelabs.pegasus-1-2-v1:0": {
"input_cost_per_video_per_second": 0.00049,
"output_cost_per_token": 7.5e-06,
"litellm_provider": "bedrock",
"mode": "chat",
"supports_video_input": true
},
"amazon.titan-text-express-v1": {
"input_cost_per_token": 1.3e-06,
"litellm_provider": "bedrock",
@@ -9066,7 +9126,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
@@ -10429,7 +10489,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+131 -74
View File
@@ -28,12 +28,16 @@ class MCPRequestHandler:
LITELLM_MCP_SERVERS_HEADER_NAME = SpecialHeaders.mcp_servers.value
LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value
# MCP Protocol Version header
MCP_PROTOCOL_VERSION_HEADER_NAME = "MCP-Protocol-Version"
@staticmethod
async def process_mcp_request(scope: Scope) -> Tuple[UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]]:
async def process_mcp_request(
scope: Scope,
) -> Tuple[
UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]]
]:
"""
Process and validate MCP request headers from the ASGI scope.
This includes:
@@ -49,7 +53,6 @@ class MCPRequestHandler:
mcp_auth_header: Optional[str] MCP auth header to be passed to the MCP server (deprecated)
mcp_servers: Optional[List[str]] List of MCP servers and access groups to use
mcp_server_auth_headers: Optional[Dict[str, str]] Server-specific auth headers in format {server_alias: auth_value}
mcp_protocol_version: Optional[str] MCP protocol version from request header
Raises:
HTTPException: If headers are invalid or missing required headers
@@ -58,39 +61,50 @@ class MCPRequestHandler:
litellm_api_key = (
MCPRequestHandler.get_litellm_api_key_from_headers(headers) or ""
)
# Get the old mcp_auth_header for backward compatibility
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
# Get the new server-specific auth headers
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
# Get MCP protocol version from header
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
# Get the new server-specific auth headers
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
# Parse MCP servers from header
mcp_servers_header = headers.get(MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME)
mcp_servers_header = headers.get(
MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME
)
verbose_logger.debug(f"Raw MCP servers header: {mcp_servers_header}")
mcp_servers = None
if mcp_servers_header is not None:
try:
mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()]
mcp_servers = [
s.strip() for s in mcp_servers_header.split(",") if s.strip()
]
verbose_logger.debug(f"Parsed MCP servers: {mcp_servers}")
except Exception as e:
verbose_logger.debug(f"Error parsing mcp_servers header: {e}")
mcp_servers = None
if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0):
if mcp_servers_header == "" or (
mcp_servers is not None and len(mcp_servers) == 0
):
mcp_servers = []
# Create a proper Request object with mock body method to avoid ASGI receive channel issues
request = Request(scope=scope)
async def mock_body():
return b"{}"
request.body = mock_body # type: ignore
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
return validated_user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return (
validated_user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
)
@staticmethod
def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]:
@@ -104,10 +118,12 @@ class MCPRequestHandler:
Support this auth: https://docs.litellm.ai/docs/mcp#using-your-mcp-with-client-side-credentials
If you want to use a different header name, you can set the `LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME` in the secret manager or `mcp_client_side_auth_header_name` in the general settings.
DEPRECATED: This method is deprecated in favor of server-specific auth headers using the format x-mcp-{{server_alias}}-{{header_name}} instead.
"""
mcp_client_side_auth_header_name: str = MCPRequestHandler._get_mcp_client_side_auth_header_name()
mcp_client_side_auth_header_name: str = (
MCPRequestHandler._get_mcp_client_side_auth_header_name()
)
auth_header = headers.get(mcp_client_side_auth_header_name)
if auth_header:
verbose_logger.warning(
@@ -115,42 +131,49 @@ class MCPRequestHandler:
f"Please use server-specific auth headers in the format 'x-mcp-{{server_alias}}-{{header_name}}' instead."
)
return auth_header
@staticmethod
def _get_mcp_server_auth_headers_from_headers(headers: Headers) -> Dict[str, str]:
"""
Parse server-specific MCP auth headers from the request headers.
Looks for headers in the format: x-mcp-{server_alias}-{header_name}
Examples:
- x-mcp-github-authorization: Bearer token123
- x-mcp-zapier-x-api-key: api_key_456
- x-mcp-deepwiki-authorization: Basic base64_encoded_creds
Returns:
Dict[str, str]: Mapping of server alias to auth value
"""
server_auth_headers = {}
prefix = "x-mcp-"
for header_name, header_value in headers.items():
if header_name.lower().startswith(prefix):
# Skip the access groups header as it's not a server auth header
if header_name.lower() == MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower() or header_name.lower() == MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower():
if (
header_name.lower()
== MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower()
or header_name.lower()
== MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower()
):
continue
# Extract server_alias and header_name from x-mcp-{server_alias}-{header_name}
remaining = header_name[len(prefix):].lower()
if '-' in remaining:
remaining = header_name[len(prefix) :].lower()
if "-" in remaining:
# Split on the last dash to separate server_alias from header_name
parts = remaining.rsplit('-', 1)
parts = remaining.rsplit("-", 1)
if len(parts) == 2:
server_alias, auth_header_name = parts
server_auth_headers[server_alias] = header_value
verbose_logger.debug(f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}...")
verbose_logger.debug(
f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}..."
)
return server_auth_headers
@staticmethod
def _get_mcp_client_side_auth_header_name() -> str:
"""
@@ -162,13 +185,21 @@ class MCPRequestHandler:
"""
from litellm.proxy.proxy_server import general_settings
from litellm.secret_managers.main import get_secret_str
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = general_settings.get("mcp_client_side_auth_header_name") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = (
MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
)
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
general_settings.get("mcp_client_side_auth_header_name")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
@staticmethod
def get_litellm_api_key_from_headers(headers: Headers) -> Optional[str]:
@@ -229,10 +260,14 @@ class MCPRequestHandler:
try:
allowed_mcp_servers: List[str] = []
allowed_mcp_servers_for_key = (
await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_key(
user_api_key_auth
)
)
allowed_mcp_servers_for_team = (
await MCPRequestHandler._get_allowed_mcp_servers_for_team(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_team(
user_api_key_auth
)
)
#########################################################
@@ -274,7 +309,9 @@ class MCPRequestHandler:
try:
key_object_permission = (
await prisma_client.db.litellm_objectpermissiontable.find_unique(
where={"object_permission_id": user_api_key_auth.object_permission_id},
where={
"object_permission_id": user_api_key_auth.object_permission_id
},
)
)
if key_object_permission is None:
@@ -282,17 +319,21 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = key_object_permission.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for key: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for key: {str(e)}"
)
return []
@staticmethod
@@ -318,10 +359,10 @@ class MCPRequestHandler:
return []
try:
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@@ -333,21 +374,27 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = object_permissions.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for team: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for team: {str(e)}"
)
return []
@staticmethod
def _get_config_server_ids_for_access_groups(config_mcp_servers, access_groups: List[str]) -> Set[str]:
def _get_config_server_ids_for_access_groups(
config_mcp_servers, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from config-loaded servers that match any of the given access groups.
"""
@@ -359,7 +406,9 @@ class MCPRequestHandler:
return server_ids
@staticmethod
async def _get_db_server_ids_for_access_groups(prisma_client, access_groups: List[str]) -> Set[str]:
async def _get_db_server_ids_for_access_groups(
prisma_client, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from DB servers that match any of the given access groups.
"""
@@ -367,21 +416,19 @@ class MCPRequestHandler:
if access_groups and prisma_client is not None:
try:
mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many(
where={
"mcp_access_groups": {
"hasSome": access_groups
}
}
where={"mcp_access_groups": {"hasSome": access_groups}}
)
for server in mcp_servers:
server_ids.add(server.server_id)
except Exception as e:
verbose_logger.debug(f"Error getting MCP servers from access groups: {e}")
verbose_logger.debug(
f"Error getting MCP servers from access groups: {e}"
)
return server_ids
@staticmethod
async def _get_mcp_servers_from_access_groups(
access_groups: List[str]
access_groups: List[str],
) -> List[str]:
"""
Resolve MCP access groups to server IDs by querying BOTH the MCP server table (DB) AND config-loaded servers
@@ -390,22 +437,28 @@ class MCPRequestHandler:
try:
# Import here to avoid circular import
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
# Use the new helper for config-loaded servers
server_ids = MCPRequestHandler._get_config_server_ids_for_access_groups(
global_mcp_server_manager.config_mcp_servers, access_groups
)
# Use the new helper for DB servers
db_server_ids = await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
db_server_ids = (
await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
)
)
server_ids.update(db_server_ids)
return list(server_ids)
except Exception as e:
verbose_logger.warning(f"Failed to get MCP servers from access groups: {str(e)}")
verbose_logger.warning(
f"Failed to get MCP servers from access groups: {str(e)}"
)
return []
@staticmethod
@@ -418,8 +471,8 @@ class MCPRequestHandler:
from typing import List
access_groups: List[str] = []
access_groups_for_key = (
await MCPRequestHandler._get_mcp_access_groups_for_key(user_api_key_auth)
access_groups_for_key = await MCPRequestHandler._get_mcp_access_groups_for_key(
user_api_key_auth
)
access_groups_for_team = (
await MCPRequestHandler._get_mcp_access_groups_for_team(user_api_key_auth)
@@ -482,10 +535,10 @@ class MCPRequestHandler:
verbose_logger.debug("prisma_client is None")
return []
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@@ -502,10 +555,14 @@ class MCPRequestHandler:
"""
Extract and parse the x-mcp-access-groups header as a list of strings.
"""
mcp_access_groups_header = headers.get(MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME)
mcp_access_groups_header = headers.get(
MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME
)
if mcp_access_groups_header is not None:
try:
return [s.strip() for s in mcp_access_groups_header.split(",") if s.strip()]
return [
s.strip() for s in mcp_access_groups_header.split(",") if s.strip()
]
except Exception:
return None
return None
@@ -516,4 +573,4 @@ class MCPRequestHandler:
Extract and parse the x-mcp-access-groups header from an ASGI scope.
"""
headers = MCPRequestHandler._safe_get_headers_from_scope(scope)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+60 -92
View File
@@ -34,8 +34,6 @@ from litellm.proxy._experimental.mcp_server.utils import (
from litellm.proxy._types import (
LiteLLM_MCPServerTable,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPTransport,
MCPTransportType,
UserAPIKeyAuth,
@@ -70,38 +68,6 @@ def _deserialize_env_dict(env_data: Any) -> Optional[Dict[str, str]]:
return env_data
def _convert_protocol_version_to_enum(
protocol_version: Optional[str | MCPSpecVersionType],
) -> MCPSpecVersionType:
"""
Convert string protocol version to MCPSpecVersion enum.
Args:
protocol_version: String protocol version, enum, or None
Returns:
MCPSpecVersionType: The enum value
"""
if not protocol_version:
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
# If it's already an MCPSpecVersion enum, return it
if isinstance(protocol_version, MCPSpecVersion):
return cast(MCPSpecVersionType, protocol_version)
# If it's a string, try to match it to enum values
if isinstance(protocol_version, str):
for version in MCPSpecVersion:
if version.value == protocol_version:
return cast(MCPSpecVersionType, version)
# If no match found, return default
verbose_logger.warning(
f"Unknown protocol version '{protocol_version}', using default"
)
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
class MCPServerManager:
def __init__(self):
self.registry: Dict[str, MCPServer] = {}
@@ -113,8 +79,7 @@ class MCPServerManager:
"name": "zapier_mcp_server",
"url": "https://actions.zapier.com/mcp/sk-ak-2ew3bofIeQIkNoeKIdXrF1Hhhp/sse"
"transport": "sse",
"auth_type": "api_key",
"spec_version": "2025-03-26"
"auth_type": "api_key"
},
"uuid-2": {
"name": "google_drive_mcp_server",
@@ -223,7 +188,6 @@ class MCPServerManager:
server_name=server_name,
url=server_config.get("url", None) or "",
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
alias=alias,
)
@@ -239,7 +203,6 @@ class MCPServerManager:
env=server_config.get("env", None) or {},
# TODO: utility fn the default values
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
authentication_token=server_config.get(
"authentication_token", server_config.get("auth_value", None)
@@ -287,7 +250,6 @@ class MCPServerManager:
server_name=getattr(mcp_server, "server_name", None),
url=mcp_server.url,
transport=cast(MCPTransportType, mcp_server.transport),
spec_version=_convert_protocol_version_to_enum(mcp_server.spec_version),
auth_type=cast(MCPAuthType, mcp_server.auth_type),
mcp_info=MCPInfo(
server_name=mcp_server.server_name or mcp_server.server_id,
@@ -350,7 +312,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all tools available across all MCP Servers.
@@ -390,7 +351,6 @@ class MCPServerManager:
tools = await self._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
list_tools_result.extend(tools)
verbose_logger.info(
@@ -414,7 +374,6 @@ class MCPServerManager:
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
protocol_version: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@@ -422,18 +381,12 @@ class MCPServerManager:
Args:
server (MCPServer): The server configuration
mcp_auth_header: MCP auth header to be passed to the MCP server. This is optional and will be used if provided.
protocol_version: Optional MCP protocol version to use. If not provided, uses server's default.
Returns:
MCPClient: Configured MCP client instance
"""
transport = server.transport or MCPTransport.sse
# Convert protocol version string to enum
protocol_version_enum = _convert_protocol_version_to_enum(
protocol_version or server.spec_version
)
# Handle stdio transport
if transport == MCPTransport.stdio:
# For stdio, we need to get the stdio config from the server
@@ -450,7 +403,6 @@ class MCPServerManager:
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
stdio_config=stdio_config,
protocol_version=protocol_version_enum,
)
else:
# For HTTP/SSE transports
@@ -461,14 +413,12 @@ class MCPServerManager:
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
protocol_version=protocol_version_enum,
)
async def _get_tools_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@@ -483,22 +433,18 @@ class MCPServerManager:
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"_get_tools_from_server for {server.name}...")
protocol_version = (
mcp_protocol_version if mcp_protocol_version else server.spec_version
)
client = None
try:
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
protocol_version=protocol_version,
)
tools = await self._fetch_tools_with_timeout(client, server.name)
prefixed_tools = self._create_prefixed_tools(tools, server)
return prefixed_tools
except Exception as e:
@@ -530,7 +476,7 @@ class MCPServerManager:
async def _list_tools_task():
try:
await client.connect()
tools = await client.list_tools()
verbose_logger.debug(f"Tools from {server_name}: {tools}")
return tools
@@ -609,7 +555,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> CallToolResult:
"""
@@ -660,32 +605,54 @@ class MCPServerManager:
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
"user_api_key_user_id": getattr(user_api_key_auth, 'user_id', None) if user_api_key_auth else None,
"user_api_key_team_id": getattr(user_api_key_auth, 'team_id', None) if user_api_key_auth else None,
"user_api_key_end_user_id": getattr(user_api_key_auth, 'end_user_id', None) if user_api_key_auth else None,
"user_api_key_hash": getattr(user_api_key_auth, 'api_key_hash', None) if user_api_key_auth else None,
"user_api_key_user_id": getattr(user_api_key_auth, "user_id", None)
if user_api_key_auth
else None,
"user_api_key_team_id": getattr(user_api_key_auth, "team_id", None)
if user_api_key_auth
else None,
"user_api_key_end_user_id": getattr(
user_api_key_auth, "end_user_id", None
)
if user_api_key_auth
else None,
"user_api_key_hash": getattr(user_api_key_auth, "api_key_hash", None)
if user_api_key_auth
else None,
}
# Create MCP request object for processing
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(pre_hook_kwargs)
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(
pre_hook_kwargs
)
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
mcp_request_obj, pre_hook_kwargs
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, #type: ignore
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = proxy_logging_obj._convert_mcp_hook_response_to_kwargs(modified_data, pre_hook_kwargs)
modified_kwargs = (
proxy_logging_obj._convert_mcp_hook_response_to_kwargs(
modified_data, pre_hook_kwargs
)
)
if modified_kwargs.get("arguments") != arguments:
arguments = modified_kwargs["arguments"]
except (BlockedPiiEntityError, GuardrailRaisedException, HTTPException) as e:
except (
BlockedPiiEntityError,
GuardrailRaisedException,
HTTPException,
) as e:
# Re-raise guardrail exceptions to properly fail the MCP call
verbose_logger.error(
f"Guardrail blocked MCP tool call pre call: {str(e)}"
@@ -706,11 +673,9 @@ class MCPServerManager:
client = self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
protocol_version=mcp_protocol_version,
)
async with client:
# Use the original tool name (without prefix) for the actual call
call_tool_params = MCPCallToolRequestParams(
name=original_tool_name,
@@ -721,7 +686,7 @@ class MCPServerManager:
# Create synthetic LLM data for during hook processing
from litellm.types.llms.base import HiddenParams
from litellm.types.mcp import MCPDuringCallRequestObject
request_obj = MCPDuringCallRequestObject(
tool_name=name,
arguments=arguments,
@@ -729,28 +694,29 @@ class MCPServerManager:
start_time=start_time.timestamp() if start_time else None,
hidden_params=HiddenParams(),
)
during_hook_kwargs = {
"name": name,
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
}
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(request_obj, during_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
request_obj, during_hook_kwargs
)
during_hook_task = asyncio.create_task(
proxy_logging_obj.during_call_hook(
user_api_key_dict=user_api_key_auth,
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
)
tasks.append(during_hook_task)
tasks.append(asyncio.create_task(client.call_tool(call_tool_params)))
try:
mcp_responses = await asyncio.gather(*tasks)
# If proxy_logging_obj is None, the tool call result is at index 0
@@ -839,19 +805,21 @@ class MCPServerManager:
)
verbose_logger.info("Loading MCP servers from database into registry...")
# perform authz check to filter the mcp servers user has access to
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
db_mcp_servers = await get_all_mcp_servers(prisma_client)
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
# ensure the global_mcp_server_manager is up to date with the db
for server in db_mcp_servers:
verbose_logger.debug(f"Adding server to registry: {server.server_id} ({server.server_name})")
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
self.add_update_server(server)
verbose_logger.info(f"Registry now contains {len(self.get_registry())} servers")
def get_mcp_server_by_id(self, server_id: str) -> Optional[MCPServer]:
@@ -869,7 +837,6 @@ class MCPServerManager:
server_name: str,
url: str,
transport: str,
spec_version: str,
auth_type: Optional[str] = None,
alias: Optional[str] = None,
) -> str:
@@ -885,7 +852,6 @@ class MCPServerManager:
server_name: Name of the server
url: Server URL
transport: Transport type (sse, http, etc.)
spec_version: MCP spec version
auth_type: Authentication type (optional)
alias: Server alias (optional)
@@ -893,7 +859,9 @@ class MCPServerManager:
A deterministic server ID string
"""
# Create a string from all the identifying parameters
params_string = f"{server_name}|{url}|{transport}|{spec_version}|{auth_type or ''}|{alias or ''}"
params_string = (
f"{server_name}|{url}|{transport}|{auth_type or ''}|{alias or ''}"
)
# Generate SHA-256 hash
hash_object = hashlib.sha256(params_string.encode("utf-8"))
@@ -1050,11 +1018,12 @@ class MCPServerManager:
alias=_server_config.alias,
url=_server_config.url,
transport=_server_config.transport,
spec_version=_server_config.spec_version,
auth_type=_server_config.auth_type,
created_at=datetime.datetime.now(),
updated_at=datetime.datetime.now(),
description=_server_config.mcp_info.get("description") if _server_config.mcp_info else None,
description=_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None,
mcp_info=_server_config.mcp_info,
mcp_access_groups=_server_config.access_groups or [],
# Stdio-specific fields
@@ -1111,7 +1080,6 @@ class MCPServerManager:
description=server.description,
url=server.url,
transport=server.transport,
spec_version=server.spec_version,
auth_type=server.auth_type,
created_at=server.created_at,
created_by=server.created_by,
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+78 -50
View File
@@ -23,7 +23,6 @@ router = APIRouter(
if MCP_AVAILABLE:
from litellm.experimental_mcp_client.client import MCPTool
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
_convert_protocol_version_to_enum,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.server import (
@@ -34,18 +33,24 @@ if MCP_AVAILABLE:
########################################################
############ MCP Server REST API Routes #################
def _get_server_auth_header(
server, mcp_server_auth_headers: Optional[Dict[str, str]], mcp_auth_header: Optional[str]
server,
mcp_server_auth_headers: Optional[Dict[str, str]],
mcp_auth_header: Optional[str],
) -> Optional[str]:
"""Helper function to get server-specific auth header with case-insensitive matching."""
if mcp_server_auth_headers and server.alias:
normalized_server_alias = server.alias.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_alias)
if server_auth is not None:
return server_auth
elif mcp_server_auth_headers and server.server_name:
normalized_server_name = server.server_name.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_name)
if server_auth is not None:
return server_auth
@@ -63,12 +68,11 @@ if MCP_AVAILABLE:
for tool in tools
]
async def _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version):
async def _get_tools_for_single_server(server, server_auth_header):
"""Helper function to get tools for a single server."""
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
return _create_tool_response_objects(tools, server.mcp_info)
@@ -104,17 +108,20 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
try:
# Extract auth headers from request
headers = request.headers
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(
headers
)
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
list_tools_result = []
error_message = None
# If server_id is specified, only query that specific server
if server_id:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
@@ -122,49 +129,67 @@ if MCP_AVAILABLE:
return {
"tools": [],
"error": "server_not_found",
"message": f"Server with id {server_id} not found"
"message": f"Server with id {server_id} not found",
}
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
list_tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
list_tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
return {
"tools": [],
"error": "server_error",
"message": f"Failed to get tools from server {server.name}: {str(e)}"
"message": f"Failed to get tools from server {server.name}: {str(e)}",
}
else:
# Query all servers
errors = []
for server in global_mcp_server_manager.get_registry().values():
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
list_tools_result.extend(tools_result)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
errors.append(f"{server.name}: {str(e)}")
continue
if errors and not list_tools_result:
error_message = "Failed to get tools from servers: " + "; ".join(errors)
error_message = "Failed to get tools from servers: " + "; ".join(
errors
)
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": error_message if error_message else "Successfully retrieved tools"
"message": error_message
if error_message
else "Successfully retrieved tools",
}
except Exception as e:
verbose_logger.exception("Unexpected error in list_tool_rest_api: %s", str(e))
verbose_logger.exception(
"Unexpected error in list_tool_rest_api: %s", str(e)
)
return {
"tools": [],
"error": "unexpected_error",
"message": f"An unexpected error occurred: {str(e)}"
"message": f"An unexpected error occurred: {str(e)}",
}
@router.post("/tools/call", dependencies=[Depends(user_api_key_auth)])
@@ -196,9 +221,9 @@ if MCP_AVAILABLE:
detail={
"error": "blocked_pii_entity",
"message": str(e),
"entity_type": getattr(e, 'entity_type', None),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"entity_type": getattr(e, "entity_type", None),
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
@@ -207,8 +232,8 @@ if MCP_AVAILABLE:
detail={
"error": "guardrail_violation",
"message": str(e),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except HTTPException as e:
# Re-raise HTTPException as-is to preserve status code and detail
@@ -220,10 +245,10 @@ if MCP_AVAILABLE:
status_code=500,
detail={
"error": "internal_server_error",
"message": f"An unexpected error occurred: {str(e)}"
}
"message": f"An unexpected error occurred: {str(e)}",
},
)
########################################################
# MCP Connection testing routes
# /health -> Test if we can connect to the MCP server
@@ -234,15 +259,15 @@ if MCP_AVAILABLE:
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
)
async def _execute_with_mcp_client(request: NewMCPServerRequest, operation):
"""
Common helper to create MCP client, execute operation, and ensure proper cleanup.
Args:
request: MCP server configuration
operation: Async function that takes a client and returns the operation result
Returns:
Operation result or error response
"""
@@ -254,15 +279,14 @@ if MCP_AVAILABLE:
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
spec_version=_convert_protocol_version_to_enum(request.spec_version),
auth_type=request.auth_type,
mcp_info=request.mcp_info,
),
mcp_auth_header=None,
)
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
return {"status": "error", "message": "An internal error has occurred."}
@@ -273,6 +297,7 @@ if MCP_AVAILABLE:
await client.disconnect()
except Exception as e:
verbose_logger.warning(f"Error disconnecting MCP client: {e}")
@router.post("/test/connection")
async def test_connection(
request: NewMCPServerRequest,
@@ -280,13 +305,13 @@ if MCP_AVAILABLE:
"""
Test if we can connect to the provided MCP server before adding it
"""
async def _test_connection_operation(client):
await client.connect()
return {"status": "ok"}
return await _execute_with_mcp_client(request, _test_connection_operation)
@router.post("/test/tools/list")
async def test_tools_list(
request: NewMCPServerRequest,
@@ -295,13 +320,16 @@ if MCP_AVAILABLE:
"""
Preview tools available from MCP server before adding it
"""
async def _list_tools_operation(client):
list_tools_result: List[MCPTool] = await client.list_tools()
model_dumped_tools: List[dict] = [tool.model_dump() for tool in list_tools_result]
model_dumped_tools: List[dict] = [
tool.model_dump() for tool in list_tools_result
]
return {
"tools": model_dumped_tools,
"error": None,
"message": "Successfully retrieved tools"
"message": "Successfully retrieved tools",
}
return await _execute_with_mcp_client(request, _list_tools_operation)
litellm/proxy/_experimental/mcp_server/server.py
+154 -70
View File
@@ -130,7 +130,9 @@ if MCP_AVAILABLE:
await _sse_session_manager_cm.__aenter__()
_SESSION_MANAGERS_INITIALIZED = True
verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!")
verbose_logger.info(
"MCP Server started with StreamableHTTP and SSE session managers!"
)
async def shutdown_session_managers():
"""Shutdown the session managers."""
@@ -171,11 +173,18 @@ if MCP_AVAILABLE:
"""
try:
# Get user authentication from context variable
user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version = (
get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_tools - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(f"MCP list_tools - MCP servers from context: {mcp_servers}")
verbose_logger.debug(
f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
@@ -186,9 +195,10 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.info(f"MCP list_tools - Successfully returned {len(tools)} tools")
verbose_logger.info(
f"MCP list_tools - Successfully returned {len(tools)} tools"
)
return tools
except Exception as e:
verbose_logger.exception(f"Error in list_tools endpoint: {str(e)}")
@@ -220,9 +230,16 @@ if MCP_AVAILABLE:
from litellm.proxy.proxy_server import proxy_config
# Validate arguments
user_api_key_auth, mcp_auth_header, _, mcp_server_auth_headers, mcp_protocol_version = get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
_,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(
f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}"
)
try:
# Create a body date for logging
body_data = {"name": name, "arguments": arguments}
@@ -249,17 +266,22 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
**data, # for logging
)
except BlockedPiiEntityError as e:
verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Blocked PII entity detected - {str(e)}", type="text")]
return [
TextContent(
text=f"Error: Blocked PII entity detected - {str(e)}", type="text"
)
]
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")]
return [
TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")
]
except HTTPException as e:
verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
@@ -287,6 +309,7 @@ if MCP_AVAILABLE:
Get the filtered MCP servers from the MCP server names
"""
from typing import Set
filtered_server_ids: Set[str] = set()
# Filter servers based on mcp_servers parameter if provided
if mcp_servers is not None:
@@ -297,7 +320,11 @@ if MCP_AVAILABLE:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
if server:
match_list = [s.lower() for s in [server.alias, server.server_name, server_id] if s is not None]
match_list = [
s.lower()
for s in [server.alias, server.server_name, server_id]
if s is not None
]
if server_or_group.lower() in match_list:
filtered_server_ids.add(server_id)
@@ -306,19 +333,23 @@ if MCP_AVAILABLE:
if not server_name_matched:
try:
access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
access_group_server_ids = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
)
)
# Only include servers that the user has access to
for server_id in access_group_server_ids:
if server_id in allowed_mcp_servers:
filtered_server_ids.add(server_id)
except Exception as e:
verbose_logger.debug(f"Could not resolve '{server_or_group}' as access group: {e}")
verbose_logger.debug(
f"Could not resolve '{server_or_group}' as access group: {e}"
)
if filtered_server_ids:
allowed_mcp_servers = list(filtered_server_ids)
return allowed_mcp_servers
async def _get_tools_from_mcp_servers(
@@ -326,7 +357,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@@ -344,7 +374,9 @@ if MCP_AVAILABLE:
return []
# Get allowed MCP servers based on user permissions
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(
user_api_key_auth
)
if mcp_servers is not None:
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
@@ -352,7 +384,6 @@ if MCP_AVAILABLE:
allowed_mcp_servers=allowed_mcp_servers,
)
# Get tools from each allowed server
all_tools = []
for server_id in allowed_mcp_servers:
@@ -375,15 +406,20 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
all_tools.extend(tools)
verbose_logger.debug(f"Successfully fetched {len(tools)} tools from server {server.name}")
verbose_logger.debug(
f"Successfully fetched {len(tools)} tools from server {server.name}"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from server {server.name}: {str(e)}")
verbose_logger.exception(
f"Error getting tools from server {server.name}: {str(e)}"
)
# Continue with other servers instead of failing completely
verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers")
verbose_logger.info(
f"Successfully fetched {len(all_tools)} tools total from all MCP servers"
)
return all_tools
async def _list_mcp_tools(
@@ -391,7 +427,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all available MCP tools.
@@ -415,11 +450,14 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers")
verbose_logger.debug(
f"Successfully fetched {len(managed_tools)} tools from managed MCP servers"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from managed MCP servers: {str(e)}")
verbose_logger.exception(
f"Error getting tools from managed MCP servers: {str(e)}"
)
# Continue with empty managed tools list instead of failing completely
# Get tools from local registry
@@ -430,10 +468,16 @@ if MCP_AVAILABLE:
# Convert local tools to MCPTool format
for tool in local_tools_raw:
# Convert from litellm.types.mcp_server.tool_registry.MCPTool to mcp.types.Tool
mcp_tool = MCPTool(name=tool.name, description=tool.description, inputSchema=tool.input_schema)
mcp_tool = MCPTool(
name=tool.name,
description=tool.description,
inputSchema=tool.input_schema,
)
local_tools.append(mcp_tool)
except Exception as e:
verbose_logger.exception(f"Error getting tools from local registry: {str(e)}")
verbose_logger.exception(
f"Error getting tools from local registry: {str(e)}"
)
# Continue with empty local tools list instead of failing completely
# Combine all tools
@@ -448,7 +492,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
**kwargs: Any,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""
@@ -456,35 +499,46 @@ if MCP_AVAILABLE:
"""
start_time = datetime.now()
if arguments is None:
raise HTTPException(status_code=400, detail="Request arguments are required")
raise HTTPException(
status_code=400, detail="Request arguments are required"
)
# Remove prefix from tool name for logging and processing
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(name)
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(
name
)
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = _get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = (
_get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
)
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get(
"litellm_logging_obj", None
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj", None)
if litellm_logging_obj:
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model = f"MCP: {name}"
# Try managed server tool first (pass the full prefixed name)
# Primary and recommended way to use MCP servers
#########################################################
mcp_server: Optional[MCPServer] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
mcp_server: Optional[
MCPServer
] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get(
"mcp_server_cost_info"
)
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
response = await _handle_managed_mcp_tool(
name=name, # Pass the full name (potentially prefixed)
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
litellm_logging_obj=litellm_logging_obj,
)
@@ -537,7 +591,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
litellm_logging_obj: Optional[Any] = None,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""Handle tool execution for managed server tools"""
@@ -577,12 +630,39 @@ if MCP_AVAILABLE:
Get the MCP servers from the path
"""
import re
mcp_servers_from_path: Optional[List[str]] = None
mcp_path_match = re.match(r"^/mcp/([^/]+/[^/]+|[^/]+)(/.*)?$", path)
# Match /mcp/<servers_and_maybe_path>
# Where servers can be comma-separated list of server names
# Server names can contain slashes (e.g., "custom_solutions/user_123")
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if mcp_path_match:
mcp_servers_str = mcp_path_match.group(1)
if mcp_servers_str:
mcp_servers_from_path = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
servers_and_path = mcp_path_match.group(1)
if servers_and_path:
# Check if it contains commas (comma-separated servers)
if ',' in servers_and_path:
# For comma-separated, look for a path at the end
# Common patterns: /tools, /chat/completions, etc.
path_match = re.search(r'/([^/,]+(?:/[^/,]+)*)$', servers_and_path)
if path_match:
# Path found at the end, remove it from servers
path_part = '/' + path_match.group(1)
servers_part = servers_and_path[:-len(path_part)]
mcp_servers_from_path = [s.strip() for s in servers_part.split(',') if s.strip()]
else:
# No path, just comma-separated servers
mcp_servers_from_path = [s.strip() for s in servers_and_path.split(',') if s.strip()]
else:
# Single server case - use regex approach for server/path separation
# This handles cases like "custom_solutions/user_123/chat/completions"
# where we want to extract "custom_solutions/user_123" as the server name
single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path)
if single_server_match:
server_name = single_server_match.group(1)
mcp_servers_from_path = [server_name]
else:
mcp_servers_from_path = [servers_and_path]
return mcp_servers_from_path
async def extract_mcp_auth_context(scope, path):
@@ -597,7 +677,6 @@ if MCP_AVAILABLE:
mcp_auth_header,
_,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
mcp_servers = mcp_servers_from_path
else:
@@ -606,11 +685,12 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers
async def handle_streamable_http_mcp(scope: Scope, receive: Receive, send: Send) -> None:
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
"""Handle MCP requests through StreamableHTTP."""
try:
path = scope.get("path", "")
@@ -619,20 +699,19 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
# Set the auth context variable for easy access in MCP functions
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
# Ensure session managers are initialized
@@ -656,7 +735,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@@ -669,19 +750,18 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
if not _SESSION_MANAGERS_INITIALIZED:
@@ -703,7 +783,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@@ -739,7 +821,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> None:
"""
Set the UserAPIKeyAuth in the auth context variable.
@@ -755,13 +836,17 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
auth_context_var.set(auth_user)
def get_auth_context() -> Tuple[
Optional[UserAPIKeyAuth], Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]
]:
def get_auth_context() -> (
Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
]
):
"""
Get the UserAPIKeyAuth from the auth context variable.
@@ -776,9 +861,8 @@ if MCP_AVAILABLE:
auth_user.mcp_auth_header,
auth_user.mcp_servers,
auth_user.mcp_server_auth_headers,
auth_user.mcp_protocol_version,
)
return None, None, None, None, None
return None, None, None, None
########################################################
############ End of Auth Context Functions #############
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_buildManifest.js
View File
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_ssgManifest.js
View File
litellm/proxy/_experimental/out/_next/static/chunks/117-a0da667066d322b6.js → litellm/proxy/_experimental/out/_next/static/chunks/117-6a9f6a591033d4a0.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/154-6f752d9e0a5e497b.js → litellm/proxy/_experimental/out/_next/static/chunks/154-b1f2a106d0e0d77b.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/220-8af5927d18414264.js → litellm/proxy/_experimental/out/_next/static/chunks/220-89d73a525e307735.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/50-bb8a11a7610535aa.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/50-d0da2dd7acce2eb9.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-df4939b4f2ec37d7.js
+1
View File
@@ -0,0 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{85210:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_1c856b', '__Inter_Fallback_1c856b'",fontStyle:"normal"},className:"__className_1c856b"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=85210)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-f4acf18888f0aa20.js
-1
View File
@@ -1 +0,0 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-0dbadf20167b786c.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-13b00ef4a072d920.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{67355:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=67355)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-f469bae327299fbb.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-7828c2c64e97362a.js → litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-d0d85032bb87ba51.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/page-338773f18570e0d6.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/page-73b19c9fbf8cc64f.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/main-7e39698e9e999d78.js → litellm/proxy/_experimental/out/_next/static/chunks/main-ad370c92406567cc.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/main-app-4f7318ae681a6d94.js → litellm/proxy/_experimental/out/_next/static/chunks/main-app-bb35cb240fbb9dca.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{78483:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(78483)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/css/2a9ba80f924f3272.css
-3
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/css/31b7f215e119031e.css → litellm/proxy/_experimental/out/_next/static/css/349654da14372cd9.css
+1 -1
View File
@@ -1 +1 @@
@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/55c55f0601d81cf3-s.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/26a46d62cd723877-s.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/97e0cb1ae144a2a9-s.woff2) format("woff2");unicode-range:u+1f??}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/581909926a08bbc8-s.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/df0a9ae256c0569c-s.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/8e9860b6e62d6359-s.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c0,u+2113,u+2c60-2c7f,u+a720-a7ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/e4af272ccee01ff0-s.p.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd}@font-face{font-family:__Inter_Fallback_b0dd8a;src:local("Arial");ascent-override:90.49%;descent-override:22.56%;line-gap-override:0.00%;size-adjust:107.06%}.__className_b0dd8a{font-family:__Inter_b0dd8a,__Inter_Fallback_b0dd8a;font-style:normal}
@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/ba9851c3c22cd980-s.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/21350d82a1f187e9-s.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/c5fe6dc8356a8c31-s.woff2) format("woff2");unicode-range:u+1f??}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/19cfc7226ec3afaa-s.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/df0a9ae256c0569c-s.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/8e9860b6e62d6359-s.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c0,u+2113,u+2c60-2c7f,u+a720-a7ff}@font-face{font-family:__Inter_1c856b;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/e4af272ccee01ff0-s.p.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd}@font-face{font-family:__Inter_Fallback_1c856b;src:local("Arial");ascent-override:90.49%;descent-override:22.56%;line-gap-override:0.00%;size-adjust:107.06%}.__className_1c856b{font-family:__Inter_1c856b,__Inter_Fallback_1c856b;font-style:normal}
litellm/proxy/_experimental/out/_next/static/css/4103fa525703177b.css
+3
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/media/19cfc7226ec3afaa-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/21350d82a1f187e9-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/26a46d62cd723877-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/55c55f0601d81cf3-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/581909926a08bbc8-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/97e0cb1ae144a2a9-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/ba9851c3c22cd980-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/_next/static/media/c5fe6dc8356a8c31-s.woff2
BIN
View File
Binary file not shown.
litellm/proxy/_experimental/out/index.html
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/index.txt
+2 -2
View File
@@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[30628,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-bb8a11a7610535aa.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","220","static/chunks/220-8af5927d18414264.js","154","static/chunks/154-6f752d9e0a5e497b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-338773f18570e0d6.js"],"default",1]
3:I[85617,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","220","static/chunks/220-89d73a525e307735.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-73b19c9fbf8cc64f.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["fhuPj8WYsuMGymIUE7Xgu",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/2a9ba80f924f3272.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null
litellm/proxy/_experimental/out/model_hub.txt
+2 -2
View File
@@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[52829,["50","static/chunks/50-bb8a11a7610535aa.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-6f752d9e0a5e497b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","418","static/chunks/app/model_hub/page-0dbadf20167b786c.js"],"default",1]
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","418","static/chunks/app/model_hub/page-13b00ef4a072d920.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["fhuPj8WYsuMGymIUE7Xgu",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/2a9ba80f924f3272.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null
litellm/proxy/_experimental/out/model_hub_table.html
+1 -1
View File
File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More